public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 03/20] Support multiple compression algs/levels/opts..
24+ messages / 3 participants
[nested] [flat]
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 24+ messages in thread
* Improve readability by using designated initializers when possible
@ 2024-02-21 15:03 Jelte Fennema-Nio <[email protected]>
2024-02-22 22:46 ` Re: Improve readability by using designated initializers when possible Jeff Davis <[email protected]>
0 siblings, 1 reply; 24+ messages in thread
From: Jelte Fennema-Nio @ 2024-02-21 15:03 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Michael Paquier <[email protected]>; Alvaro Herrera <[email protected]>
Usage of designated initializers came up in:
https://www.postgresql.org/message-id/flat/ZdWXhAt9Tz4d-lut%40paquier.xyz#9dc17e604e58569ad35643672b...
This converts all arrays that I could find that could clearly benefit
from this without any other code changes being necessary.
There were a few arrays that I didn't convert that seemed like they
could be useful to convert, but where the variables started counting
at 1. So by converting them elements the array would grow and elements
would be shifted by one. Changing those might be nice, but would
require some more code changes so I didn't want to combine it with
these simpler refactors. The arrays I'm talking about were
specifically tsearch_op_priority, BT_implies_table, BT_refutes_table,
and BT_implic_table.
Attachments:
[application/octet-stream] v1-0001-Use-designated-initializer-syntax-to-improve-read.patch (36.1K, ../../CAGECzQT3caUbcCcszNewCCmMbCuyP7XNAm60J3ybd6PN5kH2Dw@mail.gmail.com/2-v1-0001-Use-designated-initializer-syntax-to-improve-read.patch)
download | inline diff:
From 73c2bf13d793f320b69f346c686fca07d6e44fc1 Mon Sep 17 00:00:00 2001
From: Jelte Fennema-Nio <[email protected]>
Date: Wed, 21 Feb 2024 15:34:38 +0100
Subject: [PATCH v1] Use designated initializer syntax to improve readability
Use C99 designated initializer syntax for array elements of various
arrays. This is less verbose, more readable and less prone to typos or
ordering mistakes.
Akin to 74a730631065 and cc150596341e.
---
src/backend/catalog/dependency.c | 82 ++++++------
src/backend/parser/parser.c | 12 +-
src/backend/utils/misc/guc_tables.c | 187 +++++++++++-----------------
src/bin/pg_dump/pg_dump_sort.c | 94 +++++++-------
src/common/encnames.c | 84 ++++++-------
src/common/relpath.c | 8 +-
src/common/wchar.c | 85 +++++++------
src/include/mb/pg_wchar.h | 6 +-
8 files changed, 254 insertions(+), 304 deletions(-)
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index df9886efc95..97a126f1cbe 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -149,47 +149,47 @@ typedef struct
* See also getObjectClass().
*/
static const Oid object_classes[] = {
- RelationRelationId, /* OCLASS_CLASS */
- ProcedureRelationId, /* OCLASS_PROC */
- TypeRelationId, /* OCLASS_TYPE */
- CastRelationId, /* OCLASS_CAST */
- CollationRelationId, /* OCLASS_COLLATION */
- ConstraintRelationId, /* OCLASS_CONSTRAINT */
- ConversionRelationId, /* OCLASS_CONVERSION */
- AttrDefaultRelationId, /* OCLASS_DEFAULT */
- LanguageRelationId, /* OCLASS_LANGUAGE */
- LargeObjectRelationId, /* OCLASS_LARGEOBJECT */
- OperatorRelationId, /* OCLASS_OPERATOR */
- OperatorClassRelationId, /* OCLASS_OPCLASS */
- OperatorFamilyRelationId, /* OCLASS_OPFAMILY */
- AccessMethodRelationId, /* OCLASS_AM */
- AccessMethodOperatorRelationId, /* OCLASS_AMOP */
- AccessMethodProcedureRelationId, /* OCLASS_AMPROC */
- RewriteRelationId, /* OCLASS_REWRITE */
- TriggerRelationId, /* OCLASS_TRIGGER */
- NamespaceRelationId, /* OCLASS_SCHEMA */
- StatisticExtRelationId, /* OCLASS_STATISTIC_EXT */
- TSParserRelationId, /* OCLASS_TSPARSER */
- TSDictionaryRelationId, /* OCLASS_TSDICT */
- TSTemplateRelationId, /* OCLASS_TSTEMPLATE */
- TSConfigRelationId, /* OCLASS_TSCONFIG */
- AuthIdRelationId, /* OCLASS_ROLE */
- AuthMemRelationId, /* OCLASS_ROLE_MEMBERSHIP */
- DatabaseRelationId, /* OCLASS_DATABASE */
- TableSpaceRelationId, /* OCLASS_TBLSPACE */
- ForeignDataWrapperRelationId, /* OCLASS_FDW */
- ForeignServerRelationId, /* OCLASS_FOREIGN_SERVER */
- UserMappingRelationId, /* OCLASS_USER_MAPPING */
- DefaultAclRelationId, /* OCLASS_DEFACL */
- ExtensionRelationId, /* OCLASS_EXTENSION */
- EventTriggerRelationId, /* OCLASS_EVENT_TRIGGER */
- ParameterAclRelationId, /* OCLASS_PARAMETER_ACL */
- PolicyRelationId, /* OCLASS_POLICY */
- PublicationNamespaceRelationId, /* OCLASS_PUBLICATION_NAMESPACE */
- PublicationRelationId, /* OCLASS_PUBLICATION */
- PublicationRelRelationId, /* OCLASS_PUBLICATION_REL */
- SubscriptionRelationId, /* OCLASS_SUBSCRIPTION */
- TransformRelationId /* OCLASS_TRANSFORM */
+ [OCLASS_CLASS] = RelationRelationId,
+ [OCLASS_PROC] = ProcedureRelationId,
+ [OCLASS_TYPE] = TypeRelationId,
+ [OCLASS_CAST] = CastRelationId,
+ [OCLASS_COLLATION] = CollationRelationId,
+ [OCLASS_CONSTRAINT] = ConstraintRelationId,
+ [OCLASS_CONVERSION] = ConversionRelationId,
+ [OCLASS_DEFAULT] = AttrDefaultRelationId,
+ [OCLASS_LANGUAGE] = LanguageRelationId,
+ [OCLASS_LARGEOBJECT] = LargeObjectRelationId,
+ [OCLASS_OPERATOR] = OperatorRelationId,
+ [OCLASS_OPCLASS] = OperatorClassRelationId,
+ [OCLASS_OPFAMILY] = OperatorFamilyRelationId,
+ [OCLASS_AM] = AccessMethodRelationId,
+ [OCLASS_AMOP] = AccessMethodOperatorRelationId,
+ [OCLASS_AMPROC] = AccessMethodProcedureRelationId,
+ [OCLASS_REWRITE] = RewriteRelationId,
+ [OCLASS_TRIGGER] = TriggerRelationId,
+ [OCLASS_SCHEMA] = NamespaceRelationId,
+ [OCLASS_STATISTIC_EXT] = StatisticExtRelationId,
+ [OCLASS_TSPARSER] = TSParserRelationId,
+ [OCLASS_TSDICT] = TSDictionaryRelationId,
+ [OCLASS_TSTEMPLATE] = TSTemplateRelationId,
+ [OCLASS_TSCONFIG] = TSConfigRelationId,
+ [OCLASS_ROLE] = AuthIdRelationId,
+ [OCLASS_ROLE_MEMBERSHIP] = AuthMemRelationId,
+ [OCLASS_DATABASE] = DatabaseRelationId,
+ [OCLASS_TBLSPACE] = TableSpaceRelationId,
+ [OCLASS_FDW] = ForeignDataWrapperRelationId,
+ [OCLASS_FOREIGN_SERVER] = ForeignServerRelationId,
+ [OCLASS_USER_MAPPING] = UserMappingRelationId,
+ [OCLASS_DEFACL] = DefaultAclRelationId,
+ [OCLASS_EXTENSION] = ExtensionRelationId,
+ [OCLASS_EVENT_TRIGGER] = EventTriggerRelationId,
+ [OCLASS_PARAMETER_ACL] = ParameterAclRelationId,
+ [OCLASS_POLICY] = PolicyRelationId,
+ [OCLASS_PUBLICATION_NAMESPACE] = PublicationNamespaceRelationId,
+ [OCLASS_PUBLICATION] = PublicationRelationId,
+ [OCLASS_PUBLICATION_REL] = PublicationRelRelationId,
+ [OCLASS_SUBSCRIPTION] = SubscriptionRelationId,
+ [OCLASS_TRANSFORM] = TransformRelationId,
};
/*
diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c
index 9ec628ecbdf..3a1fa91c1b6 100644
--- a/src/backend/parser/parser.c
+++ b/src/backend/parser/parser.c
@@ -56,12 +56,12 @@ raw_parser(const char *str, RawParseMode mode)
{
/* this array is indexed by RawParseMode enum */
static const int mode_token[] = {
- 0, /* RAW_PARSE_DEFAULT */
- MODE_TYPE_NAME, /* RAW_PARSE_TYPE_NAME */
- MODE_PLPGSQL_EXPR, /* RAW_PARSE_PLPGSQL_EXPR */
- MODE_PLPGSQL_ASSIGN1, /* RAW_PARSE_PLPGSQL_ASSIGN1 */
- MODE_PLPGSQL_ASSIGN2, /* RAW_PARSE_PLPGSQL_ASSIGN2 */
- MODE_PLPGSQL_ASSIGN3 /* RAW_PARSE_PLPGSQL_ASSIGN3 */
+ [RAW_PARSE_DEFAULT] = 0,
+ [RAW_PARSE_TYPE_NAME] = MODE_TYPE_NAME,
+ [RAW_PARSE_PLPGSQL_EXPR] = MODE_PLPGSQL_EXPR,
+ [RAW_PARSE_PLPGSQL_ASSIGN1] = MODE_PLPGSQL_ASSIGN1,
+ [RAW_PARSE_PLPGSQL_ASSIGN2] = MODE_PLPGSQL_ASSIGN2,
+ [RAW_PARSE_PLPGSQL_ASSIGN3] = MODE_PLPGSQL_ASSIGN3,
};
yyextra.have_lookahead = true;
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 70652f0a3fc..be5f846bf52 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -626,13 +626,13 @@ bool in_hot_standby_guc;
*/
const char *const GucContext_Names[] =
{
- /* PGC_INTERNAL */ "internal",
- /* PGC_POSTMASTER */ "postmaster",
- /* PGC_SIGHUP */ "sighup",
- /* PGC_SU_BACKEND */ "superuser-backend",
- /* PGC_BACKEND */ "backend",
- /* PGC_SUSET */ "superuser",
- /* PGC_USERSET */ "user"
+ [PGC_INTERNAL] = "internal",
+ [PGC_POSTMASTER] = "postmaster",
+ [PGC_SIGHUP] = "sighup",
+ [PGC_SU_BACKEND] = "superuser-backend",
+ [PGC_BACKEND] = "backend",
+ [PGC_SUSET] = "superuser",
+ [PGC_USERSET] = "user"
};
StaticAssertDecl(lengthof(GucContext_Names) == (PGC_USERSET + 1),
@@ -645,20 +645,20 @@ StaticAssertDecl(lengthof(GucContext_Names) == (PGC_USERSET + 1),
*/
const char *const GucSource_Names[] =
{
- /* PGC_S_DEFAULT */ "default",
- /* PGC_S_DYNAMIC_DEFAULT */ "default",
- /* PGC_S_ENV_VAR */ "environment variable",
- /* PGC_S_FILE */ "configuration file",
- /* PGC_S_ARGV */ "command line",
- /* PGC_S_GLOBAL */ "global",
- /* PGC_S_DATABASE */ "database",
- /* PGC_S_USER */ "user",
- /* PGC_S_DATABASE_USER */ "database user",
- /* PGC_S_CLIENT */ "client",
- /* PGC_S_OVERRIDE */ "override",
- /* PGC_S_INTERACTIVE */ "interactive",
- /* PGC_S_TEST */ "test",
- /* PGC_S_SESSION */ "session"
+ [PGC_S_DEFAULT] = "default",
+ [PGC_S_DYNAMIC_DEFAULT] = "default",
+ [PGC_S_ENV_VAR] = "environment variable",
+ [PGC_S_FILE] = "configuration file",
+ [PGC_S_ARGV] = "command line",
+ [PGC_S_GLOBAL] = "global",
+ [PGC_S_DATABASE] = "database",
+ [PGC_S_USER] = "user",
+ [PGC_S_DATABASE_USER] = "database user",
+ [PGC_S_CLIENT] = "client",
+ [PGC_S_OVERRIDE] = "override",
+ [PGC_S_INTERACTIVE] = "interactive",
+ [PGC_S_TEST] = "test",
+ [PGC_S_SESSION] = "session"
};
StaticAssertDecl(lengthof(GucSource_Names) == (PGC_S_SESSION + 1),
@@ -669,96 +669,51 @@ StaticAssertDecl(lengthof(GucSource_Names) == (PGC_S_SESSION + 1),
*/
const char *const config_group_names[] =
{
- /* UNGROUPED */
- gettext_noop("Ungrouped"),
- /* FILE_LOCATIONS */
- gettext_noop("File Locations"),
- /* CONN_AUTH_SETTINGS */
- gettext_noop("Connections and Authentication / Connection Settings"),
- /* CONN_AUTH_TCP */
- gettext_noop("Connections and Authentication / TCP Settings"),
- /* CONN_AUTH_AUTH */
- gettext_noop("Connections and Authentication / Authentication"),
- /* CONN_AUTH_SSL */
- gettext_noop("Connections and Authentication / SSL"),
- /* RESOURCES_MEM */
- gettext_noop("Resource Usage / Memory"),
- /* RESOURCES_DISK */
- gettext_noop("Resource Usage / Disk"),
- /* RESOURCES_KERNEL */
- gettext_noop("Resource Usage / Kernel Resources"),
- /* RESOURCES_VACUUM_DELAY */
- gettext_noop("Resource Usage / Cost-Based Vacuum Delay"),
- /* RESOURCES_BGWRITER */
- gettext_noop("Resource Usage / Background Writer"),
- /* RESOURCES_ASYNCHRONOUS */
- gettext_noop("Resource Usage / Asynchronous Behavior"),
- /* WAL_SETTINGS */
- gettext_noop("Write-Ahead Log / Settings"),
- /* WAL_CHECKPOINTS */
- gettext_noop("Write-Ahead Log / Checkpoints"),
- /* WAL_ARCHIVING */
- gettext_noop("Write-Ahead Log / Archiving"),
- /* WAL_RECOVERY */
- gettext_noop("Write-Ahead Log / Recovery"),
- /* WAL_ARCHIVE_RECOVERY */
- gettext_noop("Write-Ahead Log / Archive Recovery"),
- /* WAL_RECOVERY_TARGET */
- gettext_noop("Write-Ahead Log / Recovery Target"),
- /* WAL_SUMMARIZATION */
- gettext_noop("Write-Ahead Log / Summarization"),
- /* REPLICATION_SENDING */
- gettext_noop("Replication / Sending Servers"),
- /* REPLICATION_PRIMARY */
- gettext_noop("Replication / Primary Server"),
- /* REPLICATION_STANDBY */
- gettext_noop("Replication / Standby Servers"),
- /* REPLICATION_SUBSCRIBERS */
- gettext_noop("Replication / Subscribers"),
- /* QUERY_TUNING_METHOD */
- gettext_noop("Query Tuning / Planner Method Configuration"),
- /* QUERY_TUNING_COST */
- gettext_noop("Query Tuning / Planner Cost Constants"),
- /* QUERY_TUNING_GEQO */
- gettext_noop("Query Tuning / Genetic Query Optimizer"),
- /* QUERY_TUNING_OTHER */
- gettext_noop("Query Tuning / Other Planner Options"),
- /* LOGGING_WHERE */
- gettext_noop("Reporting and Logging / Where to Log"),
- /* LOGGING_WHEN */
- gettext_noop("Reporting and Logging / When to Log"),
- /* LOGGING_WHAT */
- gettext_noop("Reporting and Logging / What to Log"),
- /* PROCESS_TITLE */
- gettext_noop("Reporting and Logging / Process Title"),
- /* STATS_MONITORING */
- gettext_noop("Statistics / Monitoring"),
- /* STATS_CUMULATIVE */
- gettext_noop("Statistics / Cumulative Query and Index Statistics"),
- /* AUTOVACUUM */
- gettext_noop("Autovacuum"),
- /* CLIENT_CONN_STATEMENT */
- gettext_noop("Client Connection Defaults / Statement Behavior"),
- /* CLIENT_CONN_LOCALE */
- gettext_noop("Client Connection Defaults / Locale and Formatting"),
- /* CLIENT_CONN_PRELOAD */
- gettext_noop("Client Connection Defaults / Shared Library Preloading"),
- /* CLIENT_CONN_OTHER */
- gettext_noop("Client Connection Defaults / Other Defaults"),
- /* LOCK_MANAGEMENT */
- gettext_noop("Lock Management"),
- /* COMPAT_OPTIONS_PREVIOUS */
- gettext_noop("Version and Platform Compatibility / Previous PostgreSQL Versions"),
- /* COMPAT_OPTIONS_CLIENT */
- gettext_noop("Version and Platform Compatibility / Other Platforms and Clients"),
- /* ERROR_HANDLING_OPTIONS */
- gettext_noop("Error Handling"),
- /* PRESET_OPTIONS */
- gettext_noop("Preset Options"),
- /* CUSTOM_OPTIONS */
- gettext_noop("Customized Options"),
- /* DEVELOPER_OPTIONS */
- gettext_noop("Developer Options"),
+ [UNGROUPED] = gettext_noop("Ungrouped"),
+ [FILE_LOCATIONS] = gettext_noop("File Locations"),
+ [CONN_AUTH_SETTINGS] = gettext_noop("Connections and Authentication / Connection Settings"),
+ [CONN_AUTH_TCP] = gettext_noop("Connections and Authentication / TCP Settings"),
+ [CONN_AUTH_AUTH] = gettext_noop("Connections and Authentication / Authentication"),
+ [CONN_AUTH_SSL] = gettext_noop("Connections and Authentication / SSL"),
+ [RESOURCES_MEM] = gettext_noop("Resource Usage / Memory"),
+ [RESOURCES_DISK] = gettext_noop("Resource Usage / Disk"),
+ [RESOURCES_KERNEL] = gettext_noop("Resource Usage / Kernel Resources"),
+ [RESOURCES_VACUUM_DELAY] = gettext_noop("Resource Usage / Cost-Based Vacuum Delay"),
+ [RESOURCES_BGWRITER] = gettext_noop("Resource Usage / Background Writer"),
+ [RESOURCES_ASYNCHRONOUS] = gettext_noop("Resource Usage / Asynchronous Behavior"),
+ [WAL_SETTINGS] = gettext_noop("Write-Ahead Log / Settings"),
+ [WAL_CHECKPOINTS] = gettext_noop("Write-Ahead Log / Checkpoints"),
+ [WAL_ARCHIVING] = gettext_noop("Write-Ahead Log / Archiving"),
+ [WAL_RECOVERY] = gettext_noop("Write-Ahead Log / Recovery"),
+ [WAL_ARCHIVE_RECOVERY] = gettext_noop("Write-Ahead Log / Archive Recovery"),
+ [WAL_RECOVERY_TARGET] = gettext_noop("Write-Ahead Log / Recovery Target"),
+ [WAL_SUMMARIZATION] = gettext_noop("Write-Ahead Log / Summarization"),
+ [REPLICATION_SENDING] = gettext_noop("Replication / Sending Servers"),
+ [REPLICATION_PRIMARY] = gettext_noop("Replication / Primary Server"),
+ [REPLICATION_STANDBY] = gettext_noop("Replication / Standby Servers"),
+ [REPLICATION_SUBSCRIBERS] = gettext_noop("Replication / Subscribers"),
+ [QUERY_TUNING_METHOD] = gettext_noop("Query Tuning / Planner Method Configuration"),
+ [QUERY_TUNING_COST] = gettext_noop("Query Tuning / Planner Cost Constants"),
+ [QUERY_TUNING_GEQO] = gettext_noop("Query Tuning / Genetic Query Optimizer"),
+ [QUERY_TUNING_OTHER] = gettext_noop("Query Tuning / Other Planner Options"),
+ [LOGGING_WHERE] = gettext_noop("Reporting and Logging / Where to Log"),
+ [LOGGING_WHEN] = gettext_noop("Reporting and Logging / When to Log"),
+ [LOGGING_WHAT] = gettext_noop("Reporting and Logging / What to Log"),
+ [PROCESS_TITLE] = gettext_noop("Reporting and Logging / Process Title"),
+ [STATS_MONITORING] = gettext_noop("Statistics / Monitoring"),
+ [STATS_CUMULATIVE] = gettext_noop("Statistics / Cumulative Query and Index Statistics"),
+ [AUTOVACUUM] = gettext_noop("Autovacuum"),
+ [CLIENT_CONN_STATEMENT] = gettext_noop("Client Connection Defaults / Statement Behavior"),
+ [CLIENT_CONN_LOCALE] = gettext_noop("Client Connection Defaults / Locale and Formatting"),
+ [CLIENT_CONN_PRELOAD] = gettext_noop("Client Connection Defaults / Shared Library Preloading"),
+ [CLIENT_CONN_OTHER] = gettext_noop("Client Connection Defaults / Other Defaults"),
+ [LOCK_MANAGEMENT] = gettext_noop("Lock Management"),
+ [COMPAT_OPTIONS_PREVIOUS] = gettext_noop("Version and Platform Compatibility / Previous PostgreSQL Versions"),
+ [COMPAT_OPTIONS_CLIENT] = gettext_noop("Version and Platform Compatibility / Other Platforms and Clients"),
+ [ERROR_HANDLING_OPTIONS] = gettext_noop("Error Handling"),
+ [PRESET_OPTIONS] = gettext_noop("Preset Options"),
+ [CUSTOM_OPTIONS] = gettext_noop("Customized Options"),
+ [DEVELOPER_OPTIONS] = gettext_noop("Developer Options"),
/* help_config wants this array to be null-terminated */
NULL
};
@@ -773,11 +728,11 @@ StaticAssertDecl(lengthof(config_group_names) == (DEVELOPER_OPTIONS + 2),
*/
const char *const config_type_names[] =
{
- /* PGC_BOOL */ "bool",
- /* PGC_INT */ "integer",
- /* PGC_REAL */ "real",
- /* PGC_STRING */ "string",
- /* PGC_ENUM */ "enum"
+ [PGC_BOOL] = "bool",
+ [PGC_INT] = "integer",
+ [PGC_REAL] = "real",
+ [PGC_STRING] = "string",
+ [PGC_ENUM] = "enum"
};
StaticAssertDecl(lengthof(config_type_names) == (PGC_ENUM + 1),
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index 8ee8a42781a..4cb754caa55 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -104,53 +104,53 @@ enum dbObjectTypePriorities
/* This table is indexed by enum DumpableObjectType */
static const int dbObjectTypePriority[] =
{
- PRIO_NAMESPACE, /* DO_NAMESPACE */
- PRIO_EXTENSION, /* DO_EXTENSION */
- PRIO_TYPE, /* DO_TYPE */
- PRIO_TYPE, /* DO_SHELL_TYPE */
- PRIO_FUNC, /* DO_FUNC */
- PRIO_AGG, /* DO_AGG */
- PRIO_OPERATOR, /* DO_OPERATOR */
- PRIO_ACCESS_METHOD, /* DO_ACCESS_METHOD */
- PRIO_OPFAMILY, /* DO_OPCLASS */
- PRIO_OPFAMILY, /* DO_OPFAMILY */
- PRIO_COLLATION, /* DO_COLLATION */
- PRIO_CONVERSION, /* DO_CONVERSION */
- PRIO_TABLE, /* DO_TABLE */
- PRIO_TABLE_ATTACH, /* DO_TABLE_ATTACH */
- PRIO_ATTRDEF, /* DO_ATTRDEF */
- PRIO_INDEX, /* DO_INDEX */
- PRIO_INDEX_ATTACH, /* DO_INDEX_ATTACH */
- PRIO_STATSEXT, /* DO_STATSEXT */
- PRIO_RULE, /* DO_RULE */
- PRIO_TRIGGER, /* DO_TRIGGER */
- PRIO_CONSTRAINT, /* DO_CONSTRAINT */
- PRIO_FK_CONSTRAINT, /* DO_FK_CONSTRAINT */
- PRIO_PROCLANG, /* DO_PROCLANG */
- PRIO_CAST, /* DO_CAST */
- PRIO_TABLE_DATA, /* DO_TABLE_DATA */
- PRIO_SEQUENCE_SET, /* DO_SEQUENCE_SET */
- PRIO_DUMMY_TYPE, /* DO_DUMMY_TYPE */
- PRIO_TSPARSER, /* DO_TSPARSER */
- PRIO_TSDICT, /* DO_TSDICT */
- PRIO_TSTEMPLATE, /* DO_TSTEMPLATE */
- PRIO_TSCONFIG, /* DO_TSCONFIG */
- PRIO_FDW, /* DO_FDW */
- PRIO_FOREIGN_SERVER, /* DO_FOREIGN_SERVER */
- PRIO_DEFAULT_ACL, /* DO_DEFAULT_ACL */
- PRIO_TRANSFORM, /* DO_TRANSFORM */
- PRIO_LARGE_OBJECT, /* DO_LARGE_OBJECT */
- PRIO_LARGE_OBJECT_DATA, /* DO_LARGE_OBJECT_DATA */
- PRIO_PRE_DATA_BOUNDARY, /* DO_PRE_DATA_BOUNDARY */
- PRIO_POST_DATA_BOUNDARY, /* DO_POST_DATA_BOUNDARY */
- PRIO_EVENT_TRIGGER, /* DO_EVENT_TRIGGER */
- PRIO_REFRESH_MATVIEW, /* DO_REFRESH_MATVIEW */
- PRIO_POLICY, /* DO_POLICY */
- PRIO_PUBLICATION, /* DO_PUBLICATION */
- PRIO_PUBLICATION_REL, /* DO_PUBLICATION_REL */
- PRIO_PUBLICATION_TABLE_IN_SCHEMA, /* DO_PUBLICATION_TABLE_IN_SCHEMA */
- PRIO_SUBSCRIPTION, /* DO_SUBSCRIPTION */
- PRIO_SUBSCRIPTION_REL /* DO_SUBSCRIPTION_REL */
+ [DO_NAMESPACE] = PRIO_NAMESPACE,
+ [DO_EXTENSION] = PRIO_EXTENSION,
+ [DO_TYPE] = PRIO_TYPE,
+ [DO_SHELL_TYPE] = PRIO_TYPE,
+ [DO_FUNC] = PRIO_FUNC,
+ [DO_AGG] = PRIO_AGG,
+ [DO_OPERATOR] = PRIO_OPERATOR,
+ [DO_ACCESS_METHOD] = PRIO_ACCESS_METHOD,
+ [DO_OPCLASS] = PRIO_OPFAMILY,
+ [DO_OPFAMILY] = PRIO_OPFAMILY,
+ [DO_COLLATION] = PRIO_COLLATION,
+ [DO_CONVERSION] = PRIO_CONVERSION,
+ [DO_TABLE] = PRIO_TABLE,
+ [DO_TABLE_ATTACH] = PRIO_TABLE_ATTACH,
+ [DO_ATTRDEF] = PRIO_ATTRDEF,
+ [DO_INDEX] = PRIO_INDEX,
+ [DO_INDEX_ATTACH] = PRIO_INDEX_ATTACH,
+ [DO_STATSEXT] = PRIO_STATSEXT,
+ [DO_RULE] = PRIO_RULE,
+ [DO_TRIGGER] = PRIO_TRIGGER,
+ [DO_CONSTRAINT] = PRIO_CONSTRAINT,
+ [DO_FK_CONSTRAINT] = PRIO_FK_CONSTRAINT,
+ [DO_PROCLANG] = PRIO_PROCLANG,
+ [DO_CAST] = PRIO_CAST,
+ [DO_TABLE_DATA] = PRIO_TABLE_DATA,
+ [DO_SEQUENCE_SET] = PRIO_SEQUENCE_SET,
+ [DO_DUMMY_TYPE] = PRIO_DUMMY_TYPE,
+ [DO_TSPARSER] = PRIO_TSPARSER,
+ [DO_TSDICT] = PRIO_TSDICT,
+ [DO_TSTEMPLATE] = PRIO_TSTEMPLATE,
+ [DO_TSCONFIG] = PRIO_TSCONFIG,
+ [DO_FDW] = PRIO_FDW,
+ [DO_FOREIGN_SERVER] = PRIO_FOREIGN_SERVER,
+ [DO_DEFAULT_ACL] = PRIO_DEFAULT_ACL,
+ [DO_TRANSFORM] = PRIO_TRANSFORM,
+ [DO_LARGE_OBJECT] = PRIO_LARGE_OBJECT,
+ [DO_LARGE_OBJECT_DATA] = PRIO_LARGE_OBJECT_DATA,
+ [DO_PRE_DATA_BOUNDARY] = PRIO_PRE_DATA_BOUNDARY,
+ [DO_POST_DATA_BOUNDARY] = PRIO_POST_DATA_BOUNDARY,
+ [DO_EVENT_TRIGGER] = PRIO_EVENT_TRIGGER,
+ [DO_REFRESH_MATVIEW] = PRIO_REFRESH_MATVIEW,
+ [DO_POLICY] = PRIO_POLICY,
+ [DO_PUBLICATION] = PRIO_PUBLICATION,
+ [DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL,
+ [DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA,
+ [DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION,
+ [DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL,
};
StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1),
diff --git a/src/common/encnames.c b/src/common/encnames.c
index 710b747f6bd..6b1132bdaf2 100644
--- a/src/common/encnames.c
+++ b/src/common/encnames.c
@@ -308,48 +308,48 @@ static const pg_encname pg_encname_tbl[] =
const pg_enc2name pg_enc2name_tbl[] =
{
- DEF_ENC2NAME(SQL_ASCII, 0),
- DEF_ENC2NAME(EUC_JP, 20932),
- DEF_ENC2NAME(EUC_CN, 20936),
- DEF_ENC2NAME(EUC_KR, 51949),
- DEF_ENC2NAME(EUC_TW, 0),
- DEF_ENC2NAME(EUC_JIS_2004, 20932),
- DEF_ENC2NAME(UTF8, 65001),
- DEF_ENC2NAME(MULE_INTERNAL, 0),
- DEF_ENC2NAME(LATIN1, 28591),
- DEF_ENC2NAME(LATIN2, 28592),
- DEF_ENC2NAME(LATIN3, 28593),
- DEF_ENC2NAME(LATIN4, 28594),
- DEF_ENC2NAME(LATIN5, 28599),
- DEF_ENC2NAME(LATIN6, 0),
- DEF_ENC2NAME(LATIN7, 0),
- DEF_ENC2NAME(LATIN8, 0),
- DEF_ENC2NAME(LATIN9, 28605),
- DEF_ENC2NAME(LATIN10, 0),
- DEF_ENC2NAME(WIN1256, 1256),
- DEF_ENC2NAME(WIN1258, 1258),
- DEF_ENC2NAME(WIN866, 866),
- DEF_ENC2NAME(WIN874, 874),
- DEF_ENC2NAME(KOI8R, 20866),
- DEF_ENC2NAME(WIN1251, 1251),
- DEF_ENC2NAME(WIN1252, 1252),
- DEF_ENC2NAME(ISO_8859_5, 28595),
- DEF_ENC2NAME(ISO_8859_6, 28596),
- DEF_ENC2NAME(ISO_8859_7, 28597),
- DEF_ENC2NAME(ISO_8859_8, 28598),
- DEF_ENC2NAME(WIN1250, 1250),
- DEF_ENC2NAME(WIN1253, 1253),
- DEF_ENC2NAME(WIN1254, 1254),
- DEF_ENC2NAME(WIN1255, 1255),
- DEF_ENC2NAME(WIN1257, 1257),
- DEF_ENC2NAME(KOI8U, 21866),
- DEF_ENC2NAME(SJIS, 932),
- DEF_ENC2NAME(BIG5, 950),
- DEF_ENC2NAME(GBK, 936),
- DEF_ENC2NAME(UHC, 949),
- DEF_ENC2NAME(GB18030, 54936),
- DEF_ENC2NAME(JOHAB, 0),
- DEF_ENC2NAME(SHIFT_JIS_2004, 932)
+ [PG_SQL_ASCII] = DEF_ENC2NAME(SQL_ASCII, 0),
+ [PG_EUC_JP] = DEF_ENC2NAME(EUC_JP, 20932),
+ [PG_EUC_CN] = DEF_ENC2NAME(EUC_CN, 20936),
+ [PG_EUC_KR] = DEF_ENC2NAME(EUC_KR, 51949),
+ [PG_EUC_TW] = DEF_ENC2NAME(EUC_TW, 0),
+ [PG_EUC_JIS_2004] = DEF_ENC2NAME(EUC_JIS_2004, 20932),
+ [PG_UTF8] = DEF_ENC2NAME(UTF8, 65001),
+ [PG_MULE_INTERNAL] = DEF_ENC2NAME(MULE_INTERNAL, 0),
+ [PG_LATIN1] = DEF_ENC2NAME(LATIN1, 28591),
+ [PG_LATIN2] = DEF_ENC2NAME(LATIN2, 28592),
+ [PG_LATIN3] = DEF_ENC2NAME(LATIN3, 28593),
+ [PG_LATIN4] = DEF_ENC2NAME(LATIN4, 28594),
+ [PG_LATIN5] = DEF_ENC2NAME(LATIN5, 28599),
+ [PG_LATIN6] = DEF_ENC2NAME(LATIN6, 0),
+ [PG_LATIN7] = DEF_ENC2NAME(LATIN7, 0),
+ [PG_LATIN8] = DEF_ENC2NAME(LATIN8, 0),
+ [PG_LATIN9] = DEF_ENC2NAME(LATIN9, 28605),
+ [PG_LATIN10] = DEF_ENC2NAME(LATIN10, 0),
+ [PG_WIN1256] = DEF_ENC2NAME(WIN1256, 1256),
+ [PG_WIN1258] = DEF_ENC2NAME(WIN1258, 1258),
+ [PG_WIN866] = DEF_ENC2NAME(WIN866, 866),
+ [PG_WIN874] = DEF_ENC2NAME(WIN874, 874),
+ [PG_KOI8R] = DEF_ENC2NAME(KOI8R, 20866),
+ [PG_WIN1251] = DEF_ENC2NAME(WIN1251, 1251),
+ [PG_WIN1252] = DEF_ENC2NAME(WIN1252, 1252),
+ [PG_ISO_8859_5] = DEF_ENC2NAME(ISO_8859_5, 28595),
+ [PG_ISO_8859_6] = DEF_ENC2NAME(ISO_8859_6, 28596),
+ [PG_ISO_8859_7] = DEF_ENC2NAME(ISO_8859_7, 28597),
+ [PG_ISO_8859_8] = DEF_ENC2NAME(ISO_8859_8, 28598),
+ [PG_WIN1250] = DEF_ENC2NAME(WIN1250, 1250),
+ [PG_WIN1253] = DEF_ENC2NAME(WIN1253, 1253),
+ [PG_WIN1254] = DEF_ENC2NAME(WIN1254, 1254),
+ [PG_WIN1255] = DEF_ENC2NAME(WIN1255, 1255),
+ [PG_WIN1257] = DEF_ENC2NAME(WIN1257, 1257),
+ [PG_KOI8U] = DEF_ENC2NAME(KOI8U, 21866),
+ [PG_SJIS] = DEF_ENC2NAME(SJIS, 932),
+ [PG_BIG5] = DEF_ENC2NAME(BIG5, 950),
+ [PG_GBK] = DEF_ENC2NAME(GBK, 936),
+ [PG_UHC] = DEF_ENC2NAME(UHC, 949),
+ [PG_GB18030] = DEF_ENC2NAME(GB18030, 54936),
+ [PG_JOHAB] = DEF_ENC2NAME(JOHAB, 0),
+ [PG_SHIFT_JIS_2004] = DEF_ENC2NAME(SHIFT_JIS_2004, 932)
};
/* ----------
diff --git a/src/common/relpath.c b/src/common/relpath.c
index b16fe19dea6..d9214f915c9 100644
--- a/src/common/relpath.c
+++ b/src/common/relpath.c
@@ -31,10 +31,10 @@
* pg_relation_size().
*/
const char *const forkNames[] = {
- "main", /* MAIN_FORKNUM */
- "fsm", /* FSM_FORKNUM */
- "vm", /* VISIBILITYMAP_FORKNUM */
- "init" /* INIT_FORKNUM */
+ [MAIN_FORKNUM] = "main",
+ [FSM_FORKNUM] = "fsm",
+ [VISIBILITYMAP_FORKNUM] = "vm",
+ [INIT_FORKNUM] = "init",
};
StaticAssertDecl(lengthof(forkNames) == (MAX_FORKNUM + 1),
diff --git a/src/common/wchar.c b/src/common/wchar.c
index 7c5ce5ca081..7e7a7507d5c 100644
--- a/src/common/wchar.c
+++ b/src/common/wchar.c
@@ -2071,52 +2071,51 @@ pg_utf8_islegal(const unsigned char *source, int length)
/*
*-------------------------------------------------------------------
* encoding info table
- * XXX must be sorted by the same order as enum pg_enc (in mb/pg_wchar.h)
*-------------------------------------------------------------------
*/
const pg_wchar_tbl pg_wchar_table[] = {
- {pg_ascii2wchar_with_len, pg_wchar2single_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifychar, pg_ascii_verifystr, 1}, /* PG_SQL_ASCII */
- {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3}, /* PG_EUC_JP */
- {pg_euccn2wchar_with_len, pg_wchar2euc_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifychar, pg_euccn_verifystr, 2}, /* PG_EUC_CN */
- {pg_euckr2wchar_with_len, pg_wchar2euc_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifychar, pg_euckr_verifystr, 3}, /* PG_EUC_KR */
- {pg_euctw2wchar_with_len, pg_wchar2euc_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifychar, pg_euctw_verifystr, 4}, /* PG_EUC_TW */
- {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3}, /* PG_EUC_JIS_2004 */
- {pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifychar, pg_utf8_verifystr, 4}, /* PG_UTF8 */
- {pg_mule2wchar_with_len, pg_wchar2mule_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifychar, pg_mule_verifystr, 4}, /* PG_MULE_INTERNAL */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN1 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN2 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN3 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN4 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN5 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN6 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN7 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN8 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN9 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_LATIN10 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1256 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1258 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN866 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN874 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_KOI8R */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1251 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1252 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* ISO-8859-5 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* ISO-8859-6 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* ISO-8859-7 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* ISO-8859-8 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1250 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1253 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1254 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1255 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_WIN1257 */
- {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, /* PG_KOI8U */
- {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2}, /* PG_SJIS */
- {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifychar, pg_big5_verifystr, 2}, /* PG_BIG5 */
- {0, 0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifychar, pg_gbk_verifystr, 2}, /* PG_GBK */
- {0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifychar, pg_uhc_verifystr, 2}, /* PG_UHC */
- {0, 0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifychar, pg_gb18030_verifystr, 4}, /* PG_GB18030 */
- {0, 0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifychar, pg_johab_verifystr, 3}, /* PG_JOHAB */
- {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2} /* PG_SHIFT_JIS_2004 */
+ [PG_SQL_ASCII] = {pg_ascii2wchar_with_len, pg_wchar2single_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifychar, pg_ascii_verifystr, 1},
+ [PG_EUC_JP] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
+ [PG_EUC_CN] = {pg_euccn2wchar_with_len, pg_wchar2euc_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifychar, pg_euccn_verifystr, 2},
+ [PG_EUC_KR] = {pg_euckr2wchar_with_len, pg_wchar2euc_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifychar, pg_euckr_verifystr, 3},
+ [PG_EUC_TW] = {pg_euctw2wchar_with_len, pg_wchar2euc_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifychar, pg_euctw_verifystr, 4},
+ [PG_EUC_JIS_2004] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
+ [PG_UTF8] = {pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifychar, pg_utf8_verifystr, 4},
+ [PG_MULE_INTERNAL] = {pg_mule2wchar_with_len, pg_wchar2mule_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifychar, pg_mule_verifystr, 4},
+ [PG_LATIN1] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN2] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN3] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN4] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN9] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_LATIN10] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1256] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1258] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN866] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN874] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_KOI8R] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1251] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1252] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_ISO_8859_5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_ISO_8859_6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_ISO_8859_7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_ISO_8859_8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1250] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1253] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1254] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1255] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_WIN1257] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_KOI8U] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
+ [PG_SJIS] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
+ [PG_BIG5] = {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifychar, pg_big5_verifystr, 2},
+ [PG_GBK] = {0, 0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifychar, pg_gbk_verifystr, 2},
+ [PG_UHC] = {0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifychar, pg_uhc_verifystr, 2},
+ [PG_GB18030] = {0, 0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifychar, pg_gb18030_verifystr, 4},
+ [PG_JOHAB] = {0, 0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifychar, pg_johab_verifystr, 3},
+ [PG_SHIFT_JIS_2004] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
};
/*
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 1d521bea24a..9248daab3af 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -224,11 +224,7 @@ typedef unsigned int pg_wchar;
/*
* PostgreSQL encoding identifiers
*
- * WARNING: the order of this enum must be same as order of entries
- * in the pg_enc2name_tbl[] array (in src/common/encnames.c), and
- * in the pg_wchar_table[] array (in src/common/wchar.c)!
- *
- * If you add some encoding don't forget to check
+ * WARNING: If you add some encoding don't forget to check
* PG_ENCODING_BE_LAST macro.
*
* PG_SQL_ASCII is default encoding and must be = 0.
base-commit: 690805ca75463a5511ddd5fa02d73928393e6c96
--
2.34.1
^ permalink raw reply [nested|flat] 24+ messages in thread
* Re: Improve readability by using designated initializers when possible
2024-02-21 15:03 Improve readability by using designated initializers when possible Jelte Fennema-Nio <[email protected]>
@ 2024-02-22 22:46 ` Jeff Davis <[email protected]>
0 siblings, 0 replies; 24+ messages in thread
From: Jeff Davis @ 2024-02-22 22:46 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; pgsql-hackers; +Cc: Michael Paquier <[email protected]>; Alvaro Herrera <[email protected]>
On Wed, 2024-02-21 at 16:03 +0100, Jelte Fennema-Nio wrote:
> Usage of designated initializers came up in:
> https://www.postgresql.org/message-id/flat/ZdWXhAt9Tz4d-lut%40paquier.xyz#9dc17e604e58569ad35643672b...
>
> This converts all arrays that I could find that could clearly benefit
> from this without any other code changes being necessary.
Looking at the object_classes array and the ObjectClass enum, I don't
quite understand the point. It seems like a way to write OCLASS_OPCLASS
instead of OperatorClassRelationId, and similar?
Am I missing something?
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 24+ messages in thread
end of thread, other threads:[~2024-02-22 22:46 UTC | newest]
Thread overview: 24+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2024-02-21 15:03 Improve readability by using designated initializers when possible Jelte Fennema-Nio <[email protected]>
2024-02-22 22:46 ` Re: Improve readability by using designated initializers when possible Jeff Davis <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox