public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 03/20] Support multiple compression algs/levels/opts.. 33+ messages / 5 participants [nested] [flat]
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format @ 2023-01-11 10:44 Melih Mutlu <[email protected]> 2023-01-11 16:34 ` Re: Allow logical replication to copy tables in binary format vignesh C <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 0 siblings, 2 replies; 33+ messages in thread From: Melih Mutlu @ 2023-01-11 10:44 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi, Thanks for your review. [email protected] <[email protected]>, 11 Oca 2023 Çar, 11:56 tarihinde şunu yazdı: > On Mon, Nov 14, 2022 8:08 PM Melih Mutlu <[email protected]> wrote: > 1. > +# Binary enabled subscription should fail > +$node_subscriber_binary->wait_for_log("ERROR: insufficient data left in > message"); > > Should it be changed to "ERROR: ( [A-Z0-9]+:)? ", like other subscription > tests. > Done. > 2. > +# Binary disabled subscription should succeed > +$node_publisher->wait_for_catchup('tap_sub'); > > If we want to wait for table synchronization to finish, should we call > wait_for_subscription_sync()? > Done. > 3. > I also think it might be better to support copy binary only for publishers > of > v16 or later. Do you plan to implement it in the patch? > Done. Thanks, -- Melih Mutlu Microsoft Attachments: [application/octet-stream] v5-0001-Allow-logical-replication-to-copy-table-in-binary.patch (24.2K, ../../CAGPVpCR2ffpiNYea-cU4sXaijCV4GHiUBf=yupKbVFM0=zWtVw@mail.gmail.com/3-v5-0001-Allow-logical-replication-to-copy-table-in-binary.patch) download | inline diff: From 966379cb41351ce885e3bd8df1a681c4a8857dc4 Mon Sep 17 00:00:00 2001 From: Melih Mutlu <[email protected]> Date: Mon, 8 Aug 2022 14:14:44 +0300 Subject: [PATCH] Allow logical replication to copy table in binary If binary option is enabled in a subscription, copy tables in binary format during table synchronization. Without this patch, table are copied in text format even if the subscription is created with binary option. When binary format is beneficial to use, allowing the subscription to copy tables in binary in table sync phase reduces the time spent on copy. Discussion: https://postgr.es/m/CAGPVpCQvAziCLknEnygY0v1-KBtg%2BOm-9JHJYZOnNPKFJPompw%40mail.gmail.com --- doc/src/sgml/logical-replication.sgml | 9 +- doc/src/sgml/ref/create_subscription.sgml | 5 +- src/backend/replication/logical/tablesync.c | 16 +- src/test/subscription/t/002_types.pl | 287 ++++++++++++++++---- src/test/subscription/t/014_binary.pl | 28 +- 5 files changed, 278 insertions(+), 67 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 54f48be87f..58f52eca3f 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -241,10 +241,11 @@ types of the columns do not need to match, as long as the text representation of the data can be converted to the target type. For example, you can replicate from a column of type <type>integer</type> to a - column of type <type>bigint</type>. The target table can also have - additional columns not provided by the published table. Any such columns - will be filled with the default value as specified in the definition of the - target table. + column of type <type>bigint</type>. Replication in binary format is type + specific and does not allow to replicate data between different types according + to its restrictions. The target table can also have additional columns not provided + by the published table. Any such columns will be filled with the default + value as specified in the definition of the target table. </para> <sect2 id="logical-replication-subscription-slot"> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index eba72c6af6..ca017c0a15 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -189,8 +189,9 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl <term><literal>binary</literal> (<type>boolean</type>)</term> <listitem> <para> - Specifies whether the subscription will request the publisher to - send the data in binary format (as opposed to text). + Specifies whether the subscription will copy the initial data to + synchronize relations in binary format and also request the publisher + to send the data in binary format too (as opposed to text). The default is <literal>false</literal>. Even when this option is enabled, only data types having binary send and receive functions will be transferred in binary. diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 38dfce7129..b9540fe3af 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -101,6 +101,7 @@ #include "catalog/pg_type.h" #include "commands/copy.h" #include "miscadmin.h" +#include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "pgstat.h" #include "replication/logicallauncher.h" @@ -1082,6 +1083,8 @@ copy_table(Relation rel) CopyFromState cstate; List *attnamelist; ParseState *pstate; + List *options = NIL; + int server_version; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1160,6 +1163,17 @@ copy_table(Relation rel) appendStringInfoString(&cmd, ") TO STDOUT"); } + + /* If the publisher is v16 or later, copy in binary format.*/ + server_version = walrcv_server_version(LogRepWorkerWalRcvConn); + if (server_version >=160000 && MySubscription->binary) + { + appendStringInfoString(&cmd, " WITH (FORMAT binary)"); + options = lappend(options, makeDefElem("format", (Node *) makeString("binary"), -1)); + } + + elog(LOG, "version: %i, %s", server_version, cmd.data); + res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL); pfree(cmd.data); if (res->status != WALRCV_OK_COPY_OUT) @@ -1176,7 +1190,7 @@ copy_table(Relation rel) NULL, false, false); attnamelist = make_copy_attnamelist(relmapentry); - cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, NIL); + cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, options); /* Do the copy */ (void) CopyFrom(cstate); diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index 6b5853b80b..0af468dec9 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -19,6 +19,11 @@ my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->start; +# Create binary subscriber node +my $node_subscriber_binary = PostgreSQL::Test::Cluster->new('subscriber_binary'); +$node_subscriber_binary->init(allows_streaming => 'logical'); +$node_subscriber_binary->start; + # Create some preexisting content on publisher my $ddl = qq( CREATE EXTENSION hstore WITH SCHEMA public; @@ -104,6 +109,100 @@ my $ddl = qq( # Setup structure on both nodes $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +# Insert initial test data +$node_publisher->safe_psql( + 'postgres', qq( + -- test_tbl_one_array_col + INSERT INTO tst_one_array (a, b) VALUES + (1, '{1, 2, 3}'), + (2, '{2, 3, 1}'); + + -- test_tbl_arrays + INSERT INTO tst_arrays (a, b, c, d) VALUES + ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), + ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'); + + -- test_tbl_single_enum + INSERT INTO tst_one_enum (a, b) VALUES + (1, 'a'), + (2, 'b'); + + -- test_tbl_enums + INSERT INTO tst_enums (a, b) VALUES + ('a', '{b, c}'), + ('b', '{c, a}'); + + -- test_tbl_single_composites + INSERT INTO tst_one_comp (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composites + INSERT INTO tst_comps (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]); + + -- test_tbl_composite_with_enums + INSERT INTO tst_comp_enum (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composite_with_enums_array + INSERT INTO tst_comp_enum_array (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]); + + -- test_tbl_composite_with_single_enums_array_in_composite + INSERT INTO tst_comp_one_enum_array (a, b) VALUES + (1, ROW(1.0, '{a, b, c}', 1)), + (2, ROW(2.0, '{a, b, c}', 2)); + + -- test_tbl_composite_with_enums_array_in_composite + INSERT INTO tst_comp_enum_what (a, b) VALUES + (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), + (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]); + + -- test_tbl_mixed_composites + INSERT INTO tst_comp_mix_array (a, b) VALUES + (ROW( + ROW(1,'a',1), + ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], + 'a', + '{a,b,NULL,c}'), + ARRAY[ + ROW( + ROW(1,'a',1), + ARRAY[ + ROW(1,'a',1)::tst_comp_basic_t, + ROW(2,'b',2)::tst_comp_basic_t, + NULL + ], + 'a', + '{a,b,c}' + )::tst_comp_mix_t + ] + ); + + -- test_tbl_range + INSERT INTO tst_range (a, b) VALUES + (1, '[1, 10]'), + (2, '[2, 20]'); + + -- test_tbl_range_array + INSERT INTO tst_range_array (a, b, c) VALUES + (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), + (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'); + + -- tst_hstore + INSERT INTO tst_hstore (a, b) VALUES + (1, '"a"=>"1"'), + (2, '"zzz"=>"foo"'); + + -- tst_dom_constr + INSERT INTO tst_dom_constr VALUES (10); +)); # Setup logical replication my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; @@ -113,89 +212,126 @@ $node_publisher->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_slot)" ); +$node_subscriber_binary->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_binary CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_binary_slot, binary = true)" +); # Wait for initial sync to finish as well $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); +$node_subscriber_binary->wait_for_subscription_sync($node_publisher, 'tap_sub_binary'); -# Insert initial test data +my $sync_check = qq( + SET timezone = '+2'; + SELECT a, b FROM tst_one_array ORDER BY a; + SELECT a, b, c, d FROM tst_arrays ORDER BY a; + SELECT a, b FROM tst_one_enum ORDER BY a; + SELECT a, b FROM tst_enums ORDER BY a; + SELECT a, b FROM tst_one_comp ORDER BY a; + SELECT a, b FROM tst_comps ORDER BY a; + SELECT a, b FROM tst_comp_enum ORDER BY a; + SELECT a, b FROM tst_comp_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_one_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_enum_what ORDER BY a; + SELECT a, b FROM tst_comp_mix_array ORDER BY a; + SELECT a, b FROM tst_range ORDER BY a; + SELECT a, b, c FROM tst_range_array ORDER BY a; + SELECT a, b FROM tst_hstore ORDER BY a; +); + +# Check the synced data on subscribers +my $result = $node_subscriber->safe_psql('postgres', $sync_check); +my $result_binary = $node_subscriber_binary->safe_psql('postgres', $sync_check); + +my $sync_result = '1|{1,2,3} +2|{2,3,1} +{1,2,3}|{a,b,c}|{1.1,2.2,3.3}|{"1 day","2 days","3 days"} +{2,3,1}|{b,c,a}|{2.2,3.3,1.1}|{00:02:00,00:03:00,00:01:00} +1|a +2|b +a|{b,c} +b|{c,a} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,"{a,b,c}",1) +2|(2,"{a,b,c}",2) +(1,"{a,b,c}",1)|{"(1,\"{a,b,c}\",1)"} +(2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} +("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +1|[1,11) +2|[2,21) +1|["2014-08-04 00:00:00+02",infinity)|{"[1,3)","[10,21)"} +2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} +1|"a"=>"1" +2|"zzz"=>"foo"'; + +is( $result, $sync_result, 'check initial sync on subscriber'); +is( $result_binary, $sync_result, 'check initial sync on subscriber in binary'); + +# Insert test data for update to apply changes $node_publisher->safe_psql( 'postgres', qq( -- test_tbl_one_array_col INSERT INTO tst_one_array (a, b) VALUES - (1, '{1, 2, 3}'), - (2, '{2, 3, 1}'), (3, '{3, 2, 1}'), (4, '{4, 3, 2}'), (5, '{5, NULL, 3}'); -- test_tbl_arrays INSERT INTO tst_arrays (a, b, c, d) VALUES - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'), ('{3, 1, 2}', '{"c", "a", "b"}', '{3.3, 1.1, 2.2}', '{"3 years", "1 year", "2 years"}'), ('{4, 1, 2}', '{"d", "a", "b"}', '{4.4, 1.1, 2.2}', '{"4 years", "1 year", "2 years"}'), ('{5, NULL, NULL}', '{"e", NULL, "b"}', '{5.5, 1.1, NULL}', '{"5 years", NULL, NULL}'); -- test_tbl_single_enum INSERT INTO tst_one_enum (a, b) VALUES - (1, 'a'), - (2, 'b'), (3, 'c'), (4, 'd'), (5, NULL); -- test_tbl_enums INSERT INTO tst_enums (a, b) VALUES - ('a', '{b, c}'), - ('b', '{c, a}'), ('c', '{b, a}'), ('d', '{c, b}'), ('e', '{d, NULL}'); -- test_tbl_single_composites INSERT INTO tst_one_comp (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, NULL, 5)); -- test_tbl_composites INSERT INTO tst_comps (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_basic_t]), (ROW(4.0, 'd', 4), ARRAY[ROW(4, 'd', 3)::tst_comp_basic_t]), (ROW(5.0, 'e', NULL), ARRAY[NULL, ROW(5, NULL, 5)::tst_comp_basic_t]); -- test_tbl_composite_with_enums INSERT INTO tst_comp_enum (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, 'e', NULL)); -- test_tbl_composite_with_enums_array INSERT INTO tst_comp_enum_array (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_enum_t]), (ROW(4.0, 'd', 3), ARRAY[ROW(3, 'd', 3)::tst_comp_enum_t]), (ROW(5.0, 'e', 3), ARRAY[ROW(3, 'e', 3)::tst_comp_enum_t, NULL]); -- test_tbl_composite_with_single_enums_array_in_composite INSERT INTO tst_comp_one_enum_array (a, b) VALUES - (1, ROW(1.0, '{a, b, c}', 1)), - (2, ROW(2.0, '{a, b, c}', 2)), (3, ROW(3.0, '{a, b, c}', 3)), (4, ROW(4.0, '{c, b, d}', 4)), (5, ROW(5.0, '{NULL, e, NULL}', 5)); -- test_tbl_composite_with_enums_array_in_composite INSERT INTO tst_comp_enum_what (a, b) VALUES - (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), - (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]), (ROW(3.0, '{c, a, b}', 1), ARRAY[ROW(3, '{c, a, b}', 1)::tst_comp_enum_array_t]), (ROW(4.0, '{c, b, d}', 4), ARRAY[ROW(4, '{c, b, d}', 4)::tst_comp_enum_array_t]), (ROW(5.0, '{c, NULL, b}', NULL), ARRAY[ROW(5, '{c, e, b}', 1)::tst_comp_enum_array_t]); @@ -203,10 +339,10 @@ $node_publisher->safe_psql( -- test_tbl_mixed_composites INSERT INTO tst_comp_mix_array (a, b) VALUES (ROW( - ROW(1,'a',1), - ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], - 'a', - '{a,b,NULL,c}'), + ROW(2,'b',2), + ARRAY[ROW(2,'b',2)::tst_comp_basic_t, ROW(3,'c',3)::tst_comp_basic_t], + 'b', + '{b,c,NULL,d}'), ARRAY[ ROW( ROW(1,'a',1), @@ -223,36 +359,29 @@ $node_publisher->safe_psql( -- test_tbl_range INSERT INTO tst_range (a, b) VALUES - (1, '[1, 10]'), - (2, '[2, 20]'), (3, '[3, 30]'), (4, '[4, 40]'), (5, '[5, 50]'); -- test_tbl_range_array INSERT INTO tst_range_array (a, b, c) VALUES - (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), - (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'), (3, tstzrange('Fri Aug 01 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[3,4]"}'), (4, tstzrange('Thu Jul 31 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[4,5]", NULL, "[40,50]"}'), (5, NULL, NULL); -- tst_hstore INSERT INTO tst_hstore (a, b) VALUES - (1, '"a"=>"1"'), - (2, '"zzz"=>"foo"'), (3, '"123"=>"321"'), (4, '"yellow horse"=>"moaned"'); -- tst_dom_constr - INSERT INTO tst_dom_constr VALUES (10); + INSERT INTO tst_dom_constr VALUES (11); )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -my $result = $node_subscriber->safe_psql( - 'postgres', qq( +my $initial_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -268,9 +397,13 @@ my $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $initial_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $initial_check); -is( $result, '1|{1,2,3} +my $initial_result = '1|{1,2,3} 2|{2,3,1} 3|{3,2,1} 4|{4,3,2} @@ -321,6 +454,7 @@ e|{d,NULL} (4,"{c,b,d}",4)|{"(4,\"{c,b,d}\",4)"} (5,"{c,NULL,b}",)|{"(5,\"{c,e,b}\",1)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[1,11) 2|[2,21) 3|[3,31) @@ -334,8 +468,10 @@ e|{d,NULL} 1|"a"=>"1" 2|"zzz"=>"foo" 3|"123"=>"321" -4|"yellow horse"=>"moaned"', - 'check replicated inserts on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $initial_result, 'check replicated inserts on subscriber'); +is( $result_binary, $initial_result, 'check replicated inserts on subscriber in binary'); # Run batch of updates $node_publisher->safe_psql( @@ -370,10 +506,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $update_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -389,9 +524,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $update_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $update_check); -is( $result, '1|{4,5,6} +my $update_result = '1|{4,5,6} 2|{2,3,1} 3|{3,2,1} 4|{4,5,6,1} @@ -442,6 +581,7 @@ e|{e,d} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")",NULL} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[100,1001) 2|[2,21) 3|[3,31) @@ -455,8 +595,10 @@ e|{e,d} 1|"updated"=>"value" 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated updates on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $update_result, 'check replicated updates on subscriber'); +is( $result_binary, $update_result, 'check replicated updates on subscriber in binary'); # Run batch of deletes $node_publisher->safe_psql( @@ -490,10 +632,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $delete_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -509,9 +650,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); -is( $result, '3|{3,2,1} +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $delete_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $delete_check); + +my $delete_result = '3|{3,2,1} 4|{4,5,6,1} 5|{4,5,6,1} {3,1,2}|{c,a,b}|{3.3,1.1,2.2}|{"3 years","1 year","2 years"} @@ -540,26 +685,58 @@ e|{e,d} (2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} 3|["2014-08-01 00:00:00+02","2014-08-04 00:00:00+02")|{"[3,5)"} 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated deletes on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $delete_result, 'check replicated deletes on subscriber'); +is( $result_binary, $delete_result, 'check replicated deletes on subscriber in binary'); # Test a domain with a constraint backed by a SQL-language function, # which needs an active snapshot in order to operate. $node_publisher->safe_psql('postgres', - "INSERT INTO tst_dom_constr VALUES (11)"); + "INSERT INTO tst_dom_constr VALUES (12)"); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); $result = $node_subscriber->safe_psql('postgres', "SELECT sum(a) FROM tst_dom_constr"); -is($result, '21', 'sql-function constraint on domain'); +is($result, '33', 'sql-function constraint on domain'); + +$result_binary = + $node_subscriber->safe_psql('postgres', + "SELECT sum(a) FROM tst_dom_constr"); +is($result_binary, '33', 'sql-function constraint on domain'); + +# Test whether subscriber fails in case of column type mismatch +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a int); + INSERT INTO tst_binary_mismatch VALUES (1); +); +$node_publisher->safe_psql('postgres', $ddl); + +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a bigint); +); +$node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION'); +$node_subscriber_binary->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub_binary REFRESH PUBLICATION'); + +# Binary enabled subscription should fail +$node_subscriber_binary->wait_for_log(qr/ERROR: ( [A-Z0-9]+:)? insufficient data left in message/); + +# Binary disabled subscription should succeed +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); $node_subscriber->stop('fast'); +$node_subscriber_binary->stop('fast'); $node_publisher->stop('fast'); done_testing(); diff --git a/src/test/subscription/t/014_binary.pl b/src/test/subscription/t/014_binary.pl index e53e23da3e..04ad6de08f 100644 --- a/src/test/subscription/t/014_binary.pl +++ b/src/test/subscription/t/014_binary.pl @@ -36,6 +36,16 @@ my $ddl = qq( $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +# Insert some content and make sure it's synced to subscriber +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO public.test_arrays (a, b, c) VALUES + ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'); + + INSERT INTO public.test_numerical (a, b, c, d) VALUES + (1, 1.2, 1.3, 10); + )); + # Configure logical replication $node_publisher->safe_psql('postgres', "CREATE PUBLICATION tpub FOR ALL TABLES"); @@ -48,27 +58,35 @@ $node_subscriber->safe_psql('postgres', # Ensure nodes are in sync with each other $node_subscriber->wait_for_subscription_sync($node_publisher, 'tsub'); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); + +is( $result, '1|1.2|1.3|10 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three}', 'check syned data on subscriber'); + # Insert some content and make sure it's replicated across $node_publisher->safe_psql( 'postgres', qq( INSERT INTO public.test_arrays (a, b, c) VALUES - ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'), ('{3,1,2}', '{1.3, 1.1, 1.2}', '{"three", "one", "two"}'); INSERT INTO public.test_numerical (a, b, c, d) VALUES - (1, 1.2, 1.3, 10), (2, 2.2, 2.3, 20), (3, 3.2, 3.3, 30); )); $node_publisher->wait_for_catchup('tsub'); -my $result = $node_subscriber->safe_psql('postgres', - "SELECT a, b, c, d FROM test_numerical ORDER BY a"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); is( $result, '1|1.2|1.3|10 2|2.2|2.3|20 -3|3.2|3.3|30', 'check replicated data on subscriber'); +3|3.2|3.3|30 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three} +{3,1,2}|{1.3,1.1,1.2}|{three,one,two}', 'check replicated data on subscriber'); # Test updates as well $node_publisher->safe_psql( -- 2.25.1 ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-01-11 16:34 ` vignesh C <[email protected]> 1 sibling, 0 replies; 33+ messages in thread From: vignesh C @ 2023-01-11 16:34 UTC (permalink / raw) To: Melih Mutlu <[email protected]>; +Cc: [email protected] <[email protected]>; Takamichi Osumi (Fujitsu) <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers On Wed, 11 Jan 2023 at 16:14, Melih Mutlu <[email protected]> wrote: > > Hi, > > Thanks for your review. > > [email protected] <[email protected]>, 11 Oca 2023 Çar, 11:56 tarihinde şunu yazdı: >> >> On Mon, Nov 14, 2022 8:08 PM Melih Mutlu <[email protected]> wrote: >> 1. >> +# Binary enabled subscription should fail >> +$node_subscriber_binary->wait_for_log("ERROR: insufficient data left in message"); >> >> Should it be changed to "ERROR: ( [A-Z0-9]+:)? ", like other subscription tests. > > > Done. > >> >> 2. >> +# Binary disabled subscription should succeed >> +$node_publisher->wait_for_catchup('tap_sub'); >> >> If we want to wait for table synchronization to finish, should we call >> wait_for_subscription_sync()? > > > Done. > >> >> 3. >> I also think it might be better to support copy binary only for publishers of >> v16 or later. Do you plan to implement it in the patch? > > > Done. For some reason CFBot is not able to apply the patch as in [1], Could you have a look and post an updated patch if required: === Applying patches on top of PostgreSQL commit ID c96de2ce1782116bd0489b1cd69ba88189a495e8 === === applying patch ./v5-0001-Allow-logical-replication-to-copy-table-in-binary.patch gpatch: **** Only garbage was found in the patch input. [1] - http://cfbot.cputube.org/patch_41_3840.log Regards, Vignesh ^ permalink raw reply [nested|flat] 33+ messages in thread
* RE: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-01-12 03:07 ` Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 1 sibling, 1 reply; 33+ messages in thread From: Takamichi Osumi (Fujitsu) @ 2023-01-12 03:07 UTC (permalink / raw) To: 'Melih Mutlu' <[email protected]>; [email protected] <[email protected]>; +Cc: Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers On Wednesday, January 11, 2023 7:45 PM Melih Mutlu <[email protected]> wrote: > Thanks for your review. > Done. Hi, minor comments on v5. (1) publisher's version check + /* If the publisher is v16 or later, copy in binary format.*/ + server_version = walrcv_server_version(LogRepWorkerWalRcvConn); + if (server_version >=160000 && MySubscription->binary) + { + appendStringInfoString(&cmd, " WITH (FORMAT binary)"); + options = lappend(options, makeDefElem("format", (Node *) makeString("binary"), -1)); + } + + elog(LOG, "version: %i, %s", server_version, cmd.data); (1-1) There is no need to log the version and the query by elog here. (1-2) Also, I suggest we can remove the server_version variable itself, because we have only one actual reference for it. There is a style that we call walrcv_server_version in the 'if condition' directly like existing codes in fetch_remote_table_info(). (1-3) Suggestion to improve comments. FROM: /* If the publisher is v16 or later, copy in binary format.*/ TO: /* If the publisher is v16 or later, copy data in the required data format. */ (2) Minor suggestion for some test code alignment. $result = $node_subscriber->safe_psql('postgres', "SELECT sum(a) FROM tst_dom_constr"); -is($result, '21', 'sql-function constraint on domain'); +is($result, '33', 'sql-function constraint on domain'); + +$result_binary = + $node_subscriber->safe_psql('postgres', + "SELECT sum(a) FROM tst_dom_constr"); +is($result_binary, '33', 'sql-function constraint on domain'); I think if we change the order of this part of check like below, then it would look more aligned with other existing test codes introduced by this patch. --- my $domain_check = 'SELECT sum(a) FROM tst_dom_constr'; $result = $node_subscriber->safe_psql('postgres', $domain_check); $result_binary = $node_subscriber->safe_psql('postgres', $domain_check); is($result, '33', 'sql-function constraint on domain'); is($result_binary, '33', 'sql-function constraint on domain in binary'); --- Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> @ 2023-01-12 08:23 ` Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 0 siblings, 1 reply; 33+ messages in thread From: Melih Mutlu @ 2023-01-12 08:23 UTC (permalink / raw) To: Takamichi Osumi (Fujitsu) <[email protected]>; +Cc: [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi, Thanks for your reviews. Takamichi Osumi (Fujitsu) <[email protected]>, 12 Oca 2023 Per, 06:07 tarihinde şunu yazdı: > On Wednesday, January 11, 2023 7:45 PM Melih Mutlu <[email protected]> > wrote: > (1-1) There is no need to log the version and the query by elog here. > (1-2) Also, I suggest we can remove the server_version variable itself, > because we have only one actual reference for it. > There is a style that we call walrcv_server_version in the > 'if condition' directly like existing codes in > fetch_remote_table_info(). > (1-3) Suggestion to improve comments. > FROM: > /* If the publisher is v16 or later, copy in binary format.*/ > TO: > /* If the publisher is v16 or later, copy data in the required data > format. */ > Forgot to remove that LOG line. Removed it now and applied other suggestions too. > I think if we change the order of this part of check like below, then > it would look more aligned with other existing test codes introduced by > this patch. > Right. Changed it to make it more aligned with the rest. Thanks, -- Melih Mutlu Microsoft Attachments: [application/octet-stream] v6-0001-Allow-logical-replication-to-copy-table-in-binary.patch (24.2K, ../../CAGPVpCS0OzZ1tx3qT1Lc83HVK2JCc=TQ1G73tbyHYDKyo+mzFQ@mail.gmail.com/3-v6-0001-Allow-logical-replication-to-copy-table-in-binary.patch) download | inline diff: From 94a56cf20a4367cd51ac63d1e41bd32cce4876ca Mon Sep 17 00:00:00 2001 From: Melih Mutlu <[email protected]> Date: Mon, 8 Aug 2022 14:14:44 +0300 Subject: [PATCH] Allow logical replication to copy table in binary If binary option is enabled in a subscription, copy tables in binary format during table synchronization. Without this patch, table are copied in text format even if the subscription is created with binary option. When binary format is beneficial to use, allowing the subscription to copy tables in binary in table sync phase reduces the time spent on copy. Discussion: https://postgr.es/m/CAGPVpCQvAziCLknEnygY0v1-KBtg%2BOm-9JHJYZOnNPKFJPompw%40mail.gmail.com --- doc/src/sgml/logical-replication.sgml | 9 +- doc/src/sgml/ref/create_subscription.sgml | 5 +- src/backend/replication/logical/tablesync.c | 13 +- src/test/subscription/t/002_types.pl | 291 ++++++++++++++++---- src/test/subscription/t/014_binary.pl | 28 +- 5 files changed, 276 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 54f48be87f..58f52eca3f 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -241,10 +241,11 @@ types of the columns do not need to match, as long as the text representation of the data can be converted to the target type. For example, you can replicate from a column of type <type>integer</type> to a - column of type <type>bigint</type>. The target table can also have - additional columns not provided by the published table. Any such columns - will be filled with the default value as specified in the definition of the - target table. + column of type <type>bigint</type>. Replication in binary format is type + specific and does not allow to replicate data between different types according + to its restrictions. The target table can also have additional columns not provided + by the published table. Any such columns will be filled with the default + value as specified in the definition of the target table. </para> <sect2 id="logical-replication-subscription-slot"> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index eba72c6af6..ca017c0a15 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -189,8 +189,9 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl <term><literal>binary</literal> (<type>boolean</type>)</term> <listitem> <para> - Specifies whether the subscription will request the publisher to - send the data in binary format (as opposed to text). + Specifies whether the subscription will copy the initial data to + synchronize relations in binary format and also request the publisher + to send the data in binary format too (as opposed to text). The default is <literal>false</literal>. Even when this option is enabled, only data types having binary send and receive functions will be transferred in binary. diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 38dfce7129..26c36cc5c3 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -101,6 +101,7 @@ #include "catalog/pg_type.h" #include "commands/copy.h" #include "miscadmin.h" +#include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "pgstat.h" #include "replication/logicallauncher.h" @@ -1082,6 +1083,7 @@ copy_table(Relation rel) CopyFromState cstate; List *attnamelist; ParseState *pstate; + List *options = NIL; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1160,6 +1162,15 @@ copy_table(Relation rel) appendStringInfoString(&cmd, ") TO STDOUT"); } + + /* If the publisher is v16 or later, copy data in the required data format. */ + if (walrcv_server_version(LogRepWorkerWalRcvConn) >=160000 && + MySubscription->binary) + { + appendStringInfoString(&cmd, " WITH (FORMAT binary)"); + options = lappend(options, makeDefElem("format", (Node *) makeString("binary"), -1)); + } + res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL); pfree(cmd.data); if (res->status != WALRCV_OK_COPY_OUT) @@ -1176,7 +1187,7 @@ copy_table(Relation rel) NULL, false, false); attnamelist = make_copy_attnamelist(relmapentry); - cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, NIL); + cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, options); /* Do the copy */ (void) CopyFrom(cstate); diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index 6b5853b80b..f1c9ef99c8 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -19,6 +19,11 @@ my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->start; +# Create binary subscriber node +my $node_subscriber_binary = PostgreSQL::Test::Cluster->new('subscriber_binary'); +$node_subscriber_binary->init(allows_streaming => 'logical'); +$node_subscriber_binary->start; + # Create some preexisting content on publisher my $ddl = qq( CREATE EXTENSION hstore WITH SCHEMA public; @@ -104,6 +109,100 @@ my $ddl = qq( # Setup structure on both nodes $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +# Insert initial test data +$node_publisher->safe_psql( + 'postgres', qq( + -- test_tbl_one_array_col + INSERT INTO tst_one_array (a, b) VALUES + (1, '{1, 2, 3}'), + (2, '{2, 3, 1}'); + + -- test_tbl_arrays + INSERT INTO tst_arrays (a, b, c, d) VALUES + ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), + ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'); + + -- test_tbl_single_enum + INSERT INTO tst_one_enum (a, b) VALUES + (1, 'a'), + (2, 'b'); + + -- test_tbl_enums + INSERT INTO tst_enums (a, b) VALUES + ('a', '{b, c}'), + ('b', '{c, a}'); + + -- test_tbl_single_composites + INSERT INTO tst_one_comp (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composites + INSERT INTO tst_comps (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]); + + -- test_tbl_composite_with_enums + INSERT INTO tst_comp_enum (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composite_with_enums_array + INSERT INTO tst_comp_enum_array (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]); + + -- test_tbl_composite_with_single_enums_array_in_composite + INSERT INTO tst_comp_one_enum_array (a, b) VALUES + (1, ROW(1.0, '{a, b, c}', 1)), + (2, ROW(2.0, '{a, b, c}', 2)); + + -- test_tbl_composite_with_enums_array_in_composite + INSERT INTO tst_comp_enum_what (a, b) VALUES + (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), + (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]); + + -- test_tbl_mixed_composites + INSERT INTO tst_comp_mix_array (a, b) VALUES + (ROW( + ROW(1,'a',1), + ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], + 'a', + '{a,b,NULL,c}'), + ARRAY[ + ROW( + ROW(1,'a',1), + ARRAY[ + ROW(1,'a',1)::tst_comp_basic_t, + ROW(2,'b',2)::tst_comp_basic_t, + NULL + ], + 'a', + '{a,b,c}' + )::tst_comp_mix_t + ] + ); + + -- test_tbl_range + INSERT INTO tst_range (a, b) VALUES + (1, '[1, 10]'), + (2, '[2, 20]'); + + -- test_tbl_range_array + INSERT INTO tst_range_array (a, b, c) VALUES + (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), + (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'); + + -- tst_hstore + INSERT INTO tst_hstore (a, b) VALUES + (1, '"a"=>"1"'), + (2, '"zzz"=>"foo"'); + + -- tst_dom_constr + INSERT INTO tst_dom_constr VALUES (10); +)); # Setup logical replication my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; @@ -113,89 +212,126 @@ $node_publisher->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_slot)" ); +$node_subscriber_binary->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_binary CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_binary_slot, binary = true)" +); # Wait for initial sync to finish as well $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); +$node_subscriber_binary->wait_for_subscription_sync($node_publisher, 'tap_sub_binary'); -# Insert initial test data +my $sync_check = qq( + SET timezone = '+2'; + SELECT a, b FROM tst_one_array ORDER BY a; + SELECT a, b, c, d FROM tst_arrays ORDER BY a; + SELECT a, b FROM tst_one_enum ORDER BY a; + SELECT a, b FROM tst_enums ORDER BY a; + SELECT a, b FROM tst_one_comp ORDER BY a; + SELECT a, b FROM tst_comps ORDER BY a; + SELECT a, b FROM tst_comp_enum ORDER BY a; + SELECT a, b FROM tst_comp_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_one_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_enum_what ORDER BY a; + SELECT a, b FROM tst_comp_mix_array ORDER BY a; + SELECT a, b FROM tst_range ORDER BY a; + SELECT a, b, c FROM tst_range_array ORDER BY a; + SELECT a, b FROM tst_hstore ORDER BY a; +); + +# Check the synced data on subscribers +my $result = $node_subscriber->safe_psql('postgres', $sync_check); +my $result_binary = $node_subscriber_binary->safe_psql('postgres', $sync_check); + +my $sync_result = '1|{1,2,3} +2|{2,3,1} +{1,2,3}|{a,b,c}|{1.1,2.2,3.3}|{"1 day","2 days","3 days"} +{2,3,1}|{b,c,a}|{2.2,3.3,1.1}|{00:02:00,00:03:00,00:01:00} +1|a +2|b +a|{b,c} +b|{c,a} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,"{a,b,c}",1) +2|(2,"{a,b,c}",2) +(1,"{a,b,c}",1)|{"(1,\"{a,b,c}\",1)"} +(2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} +("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +1|[1,11) +2|[2,21) +1|["2014-08-04 00:00:00+02",infinity)|{"[1,3)","[10,21)"} +2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} +1|"a"=>"1" +2|"zzz"=>"foo"'; + +is( $result, $sync_result, 'check initial sync on subscriber'); +is( $result_binary, $sync_result, 'check initial sync on subscriber in binary'); + +# Insert test data for update to apply changes $node_publisher->safe_psql( 'postgres', qq( -- test_tbl_one_array_col INSERT INTO tst_one_array (a, b) VALUES - (1, '{1, 2, 3}'), - (2, '{2, 3, 1}'), (3, '{3, 2, 1}'), (4, '{4, 3, 2}'), (5, '{5, NULL, 3}'); -- test_tbl_arrays INSERT INTO tst_arrays (a, b, c, d) VALUES - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'), ('{3, 1, 2}', '{"c", "a", "b"}', '{3.3, 1.1, 2.2}', '{"3 years", "1 year", "2 years"}'), ('{4, 1, 2}', '{"d", "a", "b"}', '{4.4, 1.1, 2.2}', '{"4 years", "1 year", "2 years"}'), ('{5, NULL, NULL}', '{"e", NULL, "b"}', '{5.5, 1.1, NULL}', '{"5 years", NULL, NULL}'); -- test_tbl_single_enum INSERT INTO tst_one_enum (a, b) VALUES - (1, 'a'), - (2, 'b'), (3, 'c'), (4, 'd'), (5, NULL); -- test_tbl_enums INSERT INTO tst_enums (a, b) VALUES - ('a', '{b, c}'), - ('b', '{c, a}'), ('c', '{b, a}'), ('d', '{c, b}'), ('e', '{d, NULL}'); -- test_tbl_single_composites INSERT INTO tst_one_comp (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, NULL, 5)); -- test_tbl_composites INSERT INTO tst_comps (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_basic_t]), (ROW(4.0, 'd', 4), ARRAY[ROW(4, 'd', 3)::tst_comp_basic_t]), (ROW(5.0, 'e', NULL), ARRAY[NULL, ROW(5, NULL, 5)::tst_comp_basic_t]); -- test_tbl_composite_with_enums INSERT INTO tst_comp_enum (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, 'e', NULL)); -- test_tbl_composite_with_enums_array INSERT INTO tst_comp_enum_array (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_enum_t]), (ROW(4.0, 'd', 3), ARRAY[ROW(3, 'd', 3)::tst_comp_enum_t]), (ROW(5.0, 'e', 3), ARRAY[ROW(3, 'e', 3)::tst_comp_enum_t, NULL]); -- test_tbl_composite_with_single_enums_array_in_composite INSERT INTO tst_comp_one_enum_array (a, b) VALUES - (1, ROW(1.0, '{a, b, c}', 1)), - (2, ROW(2.0, '{a, b, c}', 2)), (3, ROW(3.0, '{a, b, c}', 3)), (4, ROW(4.0, '{c, b, d}', 4)), (5, ROW(5.0, '{NULL, e, NULL}', 5)); -- test_tbl_composite_with_enums_array_in_composite INSERT INTO tst_comp_enum_what (a, b) VALUES - (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), - (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]), (ROW(3.0, '{c, a, b}', 1), ARRAY[ROW(3, '{c, a, b}', 1)::tst_comp_enum_array_t]), (ROW(4.0, '{c, b, d}', 4), ARRAY[ROW(4, '{c, b, d}', 4)::tst_comp_enum_array_t]), (ROW(5.0, '{c, NULL, b}', NULL), ARRAY[ROW(5, '{c, e, b}', 1)::tst_comp_enum_array_t]); @@ -203,10 +339,10 @@ $node_publisher->safe_psql( -- test_tbl_mixed_composites INSERT INTO tst_comp_mix_array (a, b) VALUES (ROW( - ROW(1,'a',1), - ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], - 'a', - '{a,b,NULL,c}'), + ROW(2,'b',2), + ARRAY[ROW(2,'b',2)::tst_comp_basic_t, ROW(3,'c',3)::tst_comp_basic_t], + 'b', + '{b,c,NULL,d}'), ARRAY[ ROW( ROW(1,'a',1), @@ -223,36 +359,29 @@ $node_publisher->safe_psql( -- test_tbl_range INSERT INTO tst_range (a, b) VALUES - (1, '[1, 10]'), - (2, '[2, 20]'), (3, '[3, 30]'), (4, '[4, 40]'), (5, '[5, 50]'); -- test_tbl_range_array INSERT INTO tst_range_array (a, b, c) VALUES - (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), - (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'), (3, tstzrange('Fri Aug 01 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[3,4]"}'), (4, tstzrange('Thu Jul 31 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[4,5]", NULL, "[40,50]"}'), (5, NULL, NULL); -- tst_hstore INSERT INTO tst_hstore (a, b) VALUES - (1, '"a"=>"1"'), - (2, '"zzz"=>"foo"'), (3, '"123"=>"321"'), (4, '"yellow horse"=>"moaned"'); -- tst_dom_constr - INSERT INTO tst_dom_constr VALUES (10); + INSERT INTO tst_dom_constr VALUES (11); )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -my $result = $node_subscriber->safe_psql( - 'postgres', qq( +my $initial_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -268,9 +397,13 @@ my $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $initial_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $initial_check); -is( $result, '1|{1,2,3} +my $initial_result = '1|{1,2,3} 2|{2,3,1} 3|{3,2,1} 4|{4,3,2} @@ -321,6 +454,7 @@ e|{d,NULL} (4,"{c,b,d}",4)|{"(4,\"{c,b,d}\",4)"} (5,"{c,NULL,b}",)|{"(5,\"{c,e,b}\",1)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[1,11) 2|[2,21) 3|[3,31) @@ -334,8 +468,10 @@ e|{d,NULL} 1|"a"=>"1" 2|"zzz"=>"foo" 3|"123"=>"321" -4|"yellow horse"=>"moaned"', - 'check replicated inserts on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $initial_result, 'check replicated inserts on subscriber'); +is( $result_binary, $initial_result, 'check replicated inserts on subscriber in binary'); # Run batch of updates $node_publisher->safe_psql( @@ -370,10 +506,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $update_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -389,9 +524,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $update_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $update_check); -is( $result, '1|{4,5,6} +my $update_result = '1|{4,5,6} 2|{2,3,1} 3|{3,2,1} 4|{4,5,6,1} @@ -442,6 +581,7 @@ e|{e,d} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")",NULL} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[100,1001) 2|[2,21) 3|[3,31) @@ -455,8 +595,10 @@ e|{e,d} 1|"updated"=>"value" 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated updates on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $update_result, 'check replicated updates on subscriber'); +is( $result_binary, $update_result, 'check replicated updates on subscriber in binary'); # Run batch of deletes $node_publisher->safe_psql( @@ -490,10 +632,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $delete_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -509,9 +650,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $delete_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $delete_check); -is( $result, '3|{3,2,1} +my $delete_result = '3|{3,2,1} 4|{4,5,6,1} 5|{4,5,6,1} {3,1,2}|{c,a,b}|{3.3,1.1,2.2}|{"3 years","1 year","2 years"} @@ -540,26 +685,56 @@ e|{e,d} (2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} 3|["2014-08-01 00:00:00+02","2014-08-04 00:00:00+02")|{"[3,5)"} 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated deletes on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $delete_result, 'check replicated deletes on subscriber'); +is( $result_binary, $delete_result, 'check replicated deletes on subscriber in binary'); # Test a domain with a constraint backed by a SQL-language function, # which needs an active snapshot in order to operate. $node_publisher->safe_psql('postgres', - "INSERT INTO tst_dom_constr VALUES (11)"); + "INSERT INTO tst_dom_constr VALUES (12)"); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); + +my $domain_check = qq(SELECT sum(a) FROM tst_dom_constr); + +$result = $node_subscriber->safe_psql('postgres', $domain_check); +$result_binary = $node_subscriber->safe_psql('postgres', $domain_check); -$result = - $node_subscriber->safe_psql('postgres', - "SELECT sum(a) FROM tst_dom_constr"); -is($result, '21', 'sql-function constraint on domain'); +is( $result, '33', 'sql-function constraint on domain'); +is( $result_binary, '33', 'sql-function constraint on domain'); + +# Test whether subscriber fails in case of column type mismatch +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a int); + INSERT INTO tst_binary_mismatch VALUES (1); +); +$node_publisher->safe_psql('postgres', $ddl); + +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a bigint); +); +$node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION'); +$node_subscriber_binary->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub_binary REFRESH PUBLICATION'); + +# Binary enabled subscription should fail +$node_subscriber_binary->wait_for_log(qr/ERROR: ( [A-Z0-9]+:)? insufficient data left in message/); + +# Binary disabled subscription should succeed +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); $node_subscriber->stop('fast'); +$node_subscriber_binary->stop('fast'); $node_publisher->stop('fast'); done_testing(); diff --git a/src/test/subscription/t/014_binary.pl b/src/test/subscription/t/014_binary.pl index e53e23da3e..04ad6de08f 100644 --- a/src/test/subscription/t/014_binary.pl +++ b/src/test/subscription/t/014_binary.pl @@ -36,6 +36,16 @@ my $ddl = qq( $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +# Insert some content and make sure it's synced to subscriber +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO public.test_arrays (a, b, c) VALUES + ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'); + + INSERT INTO public.test_numerical (a, b, c, d) VALUES + (1, 1.2, 1.3, 10); + )); + # Configure logical replication $node_publisher->safe_psql('postgres', "CREATE PUBLICATION tpub FOR ALL TABLES"); @@ -48,27 +58,35 @@ $node_subscriber->safe_psql('postgres', # Ensure nodes are in sync with each other $node_subscriber->wait_for_subscription_sync($node_publisher, 'tsub'); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); + +is( $result, '1|1.2|1.3|10 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three}', 'check syned data on subscriber'); + # Insert some content and make sure it's replicated across $node_publisher->safe_psql( 'postgres', qq( INSERT INTO public.test_arrays (a, b, c) VALUES - ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'), ('{3,1,2}', '{1.3, 1.1, 1.2}', '{"three", "one", "two"}'); INSERT INTO public.test_numerical (a, b, c, d) VALUES - (1, 1.2, 1.3, 10), (2, 2.2, 2.3, 20), (3, 3.2, 3.3, 30); )); $node_publisher->wait_for_catchup('tsub'); -my $result = $node_subscriber->safe_psql('postgres', - "SELECT a, b, c, d FROM test_numerical ORDER BY a"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); is( $result, '1|1.2|1.3|10 2|2.2|2.3|20 -3|3.2|3.3|30', 'check replicated data on subscriber'); +3|3.2|3.3|30 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three} +{3,1,2}|{1.3,1.1,1.2}|{three,one,two}', 'check replicated data on subscriber'); # Test updates as well $node_publisher->safe_psql( -- 2.25.1 ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-01-18 07:16 ` Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 0 siblings, 1 reply; 33+ messages in thread From: Bharath Rupireddy @ 2023-01-18 07:16 UTC (permalink / raw) To: Melih Mutlu <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers On Thu, Jan 12, 2023 at 1:53 PM Melih Mutlu <[email protected]> wrote: > > Hi, > > Thanks for your reviews. Thanks. I have some comments: 1. The performance numbers posted upthread [1] look impressive for the use-case tried, that's a 2.25X improvement or 55.6% reduction in execution times. However, it'll be great to run a few more varied tests to confirm the benefit. 2. It'll be great to capture the perf report to see if the time spent in copy_table() is reduced with the patch. 3. I think blending initial table sync's binary copy option with data-type level binary send/receive is not good. Moreover, data-type level binary send/receive has its own restrictions per 9de77b5453. IMO, it'll be good to have a new option, say copy_data_format synonyms with COPY command's FORMAT option but with only text (default) and binary values. 4. Why to add tests to existing 002_types.pl? Can we add a new file with all the data types covered? 5. It's not clear to me as to why these rows were removed in the patch? - (1, '{1, 2, 3}'), - (2, '{2, 3, 1}'), (3, '{3, 2, 1}'), (4, '{4, 3, 2}'), (5, '{5, NULL, 3}'); -- test_tbl_arrays INSERT INTO tst_arrays (a, b, c, d) VALUES - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'), 6. BTW, the subbinary description is missing in pg_subscription docs https://www.postgresql.org/docs/devel/catalog-pg-subscription.html? - Specifies whether the subscription will request the publisher to - send the data in binary format (as opposed to text). + Specifies whether the subscription will copy the initial data to + synchronize relations in binary format and also request the publisher + to send the data in binary format too (as opposed to text). 7. A nitpick - space is needed after >= before 160000. + if (walrcv_server_version(LogRepWorkerWalRcvConn) >=160000 && 8. Note that the COPY binary format isn't portable across platforms (Windows to Linux for instance) or major versions https://www.postgresql.org/docs/devel/sql-copy.html, whereas, logical replication is https://www.postgresql.org/docs/devel/logical-replication.html. I don't see any handling of such cases in copy_table but only a check for the publisher version. I think we need to account for all the cases - allow binary COPY only when publisher and subscriber are of same versions, architectures, platforms. The trick here is how we identify if the subscriber is of the same type and taste (architectures and platforms) as the publisher. Looking for PG_VERSION_STR of publisher and subscriber might be naive, but I'm not sure if there's a better way to do it. Also, the COPY binary format is very data type specific - per the docs "for example it will not work to output binary data from a smallint column and read it into an integer column, even though that would work fine in text format.". I did a small experiment [2], the logical replication works with compatible data types (int -> smallint, even int -> text), whereas the COPY binary format doesn't. I think it'll complicate things a bit to account for the above cases and allow COPY with binary format for logical replication. [1] https://www.postgresql.org/message-id/CAGPVpCQEKDVKQPf6OFQ-9WiRYB1YRejm--YJTuwgzuvj1LEJ2A%40mail.gma... [2] DROP TABLE foo; DROP PUBLICATION mypub; CREATE TABLE foo(c1 bigint, c2 int, c3 smallint); INSERT INTO foo SELECT i , i+1, i+2 FROM generate_series(1, 5) i; CREATE PUBLICATION mypub FOR TABLE foo; SELECT COUNT(*) FROM foo; SELECT * FROM foo; DROP SUBSCRIPTION mysub; DROP TABLE foo; CREATE TABLE foo(c1 smallint, c2 smallint, c3 smallint); -- works without any problem -- OR CREATE TABLE foo(c1 smallint, c2 text, c3 smallint); -- works without any problem CREATE SUBSCRIPTION mysub CONNECTION 'port=5432 dbname=postgres user=ubuntu' PUBLICATION mypub; SELECT COUNT(*) FROM foo; SELECT * FROM foo; drop table foo; create table foo(c1 bigint, c2 int, c3 smallint); insert into foo select i, i+1, i+2 from generate_series(1, 10) i; copy foo(c1, c2, c3) to '/home/ubuntu/postgres/inst/bin/data/foo.text' with (format 'text'); copy foo(c1, c2, c3) to '/home/ubuntu/postgres/inst/bin/data/foo.binary' with (format 'binary'); drop table bar; create table bar(c1 smallint, c2 smallint, c3 smallint); -- or create table bar(c1 smallint, c2 text, c3 smallint); copy bar(c1, c2, c3) from '/home/ubuntu/postgres/inst/bin/data/foo.text' with (format 'text'); copy bar(c1, c2, c3) from '/home/ubuntu/postgres/inst/bin/data/foo.binary' with (format 'binary'); -- produces "ERROR: incorrect binary data format" -- Bharath Rupireddy PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> @ 2023-01-30 10:49 ` Melih Mutlu <[email protected]> 2023-01-30 12:34 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-02-01 03:05 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-02-16 05:16 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 0 siblings, 3 replies; 33+ messages in thread From: Melih Mutlu @ 2023-01-30 10:49 UTC (permalink / raw) To: Bharath Rupireddy <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi Bharath, Thanks for reviewing. Bharath Rupireddy <[email protected]>, 18 Oca 2023 Çar, 10:17 tarihinde şunu yazdı: > On Thu, Jan 12, 2023 at 1:53 PM Melih Mutlu <[email protected]> > wrote: > 1. The performance numbers posted upthread [1] look impressive for the > use-case tried, that's a 2.25X improvement or 55.6% reduction in > execution times. However, it'll be great to run a few more varied > tests to confirm the benefit. > Sure, do you have any specific test case or suggestion in mind? > 2. It'll be great to capture the perf report to see if the time spent > in copy_table() is reduced with the patch. > Will provide that in another email soon. > 3. I think blending initial table sync's binary copy option with > data-type level binary send/receive is not good. Moreover, data-type > level binary send/receive has its own restrictions per 9de77b5453. > IMO, it'll be good to have a new option, say copy_data_format synonyms > with COPY command's FORMAT option but with only text (default) and > binary values. > Added a "copy_format" option for subscriptions with text as default value. So it would be possible to create a binary subscription but copy tables in text format to avoid restrictions that you're concerned about. > 4. Why to add tests to existing 002_types.pl? Can we add a new file > with all the data types covered? > Since 002_types.pl is where the replication of data types are covered. I thought it would be okay to test replication with the binary option in that file. Sure, we can add a new file with different data types for testing subscriptions with binary option. But then I feel like it would have too many duplicated lines with 002_types.pl. If you think that 002_types.pl lacks some data types needs to be tested, then we should add those into 002_types.pl too whether we test subscription with binary option in that file or some other place, right? > 5. It's not clear to me as to why these rows were removed in the patch? > - (1, '{1, 2, 3}'), > - (2, '{2, 3, 1}'), > (3, '{3, 2, 1}'), > (4, '{4, 3, 2}'), > (5, '{5, NULL, 3}'); > > -- test_tbl_arrays > INSERT INTO tst_arrays (a, b, c, d) VALUES > - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 > day", "2 days", "3 days"}'), > - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 > minutes", "3 minutes", "1 minute"}'), > Previously, it wasn't actually testing the initial table sync since all tables were empty when subscription was created. I just simply split the data initially inserted to test initial table sync. With this patch, it inserts the first two rows for all data types before subscriptions get created. You can see these lines: > +# Insert initial test data > +$node_publisher->safe_psql( > + 'postgres', qq( > + -- test_tbl_one_array_col > + INSERT INTO tst_one_array (a, b) VALUES > + (1, '{1, 2, 3}'), > + (2, '{2, 3, 1}'); > + > + -- test_tbl_arrays > + INSERT INTO tst_arrays (a, b, c, d) VALUES > 6. BTW, the subbinary description is missing in pg_subscription docs > https://www.postgresql.org/docs/devel/catalog-pg-subscription.html? > - Specifies whether the subscription will request the publisher to > - send the data in binary format (as opposed to text). > + Specifies whether the subscription will copy the initial data to > + synchronize relations in binary format and also request the > publisher > + to send the data in binary format too (as opposed to text). > Done. > 7. A nitpick - space is needed after >= before 160000. > + if (walrcv_server_version(LogRepWorkerWalRcvConn) >=160000 && > Done. > 8. Note that the COPY binary format isn't portable across platforms > (Windows to Linux for instance) or major versions > https://www.postgresql.org/docs/devel/sql-copy.html, whereas, logical > replication is > https://www.postgresql.org/docs/devel/logical-replication.html. > I don't see any handling of such cases in copy_table but only a check > for the publisher version. I think we need to account for all the > cases - allow binary COPY only when publisher and subscriber are of > same versions, architectures, platforms. The trick here is how we > identify if the subscriber is of the same type and taste > (architectures and platforms) as the publisher. Looking for > PG_VERSION_STR of publisher and subscriber might be naive, but I'm not > sure if there's a better way to do it. > I think having the "copy_format" option with text as default, like I replied to your 3rd review above, will keep things as they are now. The patch now will only allow users to choose binary copy as well, if they want it and acknowledge the limitations that come with binary copy. COPY command's portability issues shouldn't be an issue right now, since the patch still supports text format. Right? > Also, the COPY binary format is very data type specific - per the docs > "for example it will not work to output binary data from a smallint > column and read it into an integer column, even though that would work > fine in text format.". I did a small experiment [2], the logical > replication works with compatible data types (int -> smallint, even > int -> text), whereas the COPY binary format doesn't. > Logical replication between different types like int and smallint is already not working properly on HEAD too. Yes, the scenario you shared looks like working. But you didn't create the subscription with binary=true. The patch did not change subscription with binary=false case. I believe what you should experiment is binary=true case which already fails in the apply phase on HEAD. Well, with this patch, it will begin to fail in the table copy phase. But I don't think this is a problem because logical replication in binary format is already broken for replications between different data types. Please see [1] and you'll get the following error in your case: "ERROR: incorrect binary data format in logical replication column 1" [1] Publisher: DROP TABLE foo; DROP PUBLICATION mypub; CREATE TABLE foo(c1 bigint, c2 int, c3 smallint); INSERT INTO foo SELECT i , i+1, i+2 FROM generate_series(1, 5) i; CREATE PUBLICATION mypub FOR TABLE foo; SELECT COUNT(*) FROM foo; SELECT * FROM foo; Subscriber: DROP SUBSCRIPTION mysub; DROP TABLE foo; CREATE TABLE foo(c1 smallint, c2 smallint, c3 smallint); CREATE SUBSCRIPTION mysub CONNECTION 'port=5432 dbname=postgres user=ubuntu' PUBLICATION mypub WITH(binary); -- table sync will be successful since they're copied in text even though set binary=true SELECT COUNT(*) FROM foo; SELECT * FROM foo; Back to publisher: INSERT INTO foo SELECT i , i+1, i+2 FROM generate_series(1, 5) i; -- insert more rows to see whether the apply also works SELECT COUNT(*) FROM foo; -- you'll see that new rows does not get replicated In subscriber logs: LOG: logical replication apply worker for subscription "mysub" has started ERROR: incorrect binary data format in logical replication column 1 CONTEXT: processing remote data for replication origin "pg_16395" during message type "INSERT" for replication target relation "public.foo" column "c1" in transaction 747, finished at 0/157F3E0 LOG: background worker "logical replication worker" (PID 16903) exited with exit code 1 Best, -- Melih Mutlu Microsoft Attachments: [application/octet-stream] v7-0001-Allow-logical-replication-to-copy-table-in-binary.patch (69.3K, ../../CAGPVpCSS_S8ZHiV+=3RvsDro16gJhVckzQDUrZ08b3LSPWTmsA@mail.gmail.com/3-v7-0001-Allow-logical-replication-to-copy-table-in-binary.patch) download | inline diff: From 0908f9d4a490010ffa2e3f69170c34406f83fffe Mon Sep 17 00:00:00 2001 From: Melih Mutlu <[email protected]> Date: Mon, 8 Aug 2022 14:14:44 +0300 Subject: [PATCH] Allow logical replication to copy table in binary If binary option is enabled in a subscription, copy tables in binary format during table synchronization. Without this patch, table are copied in text format even if the subscription is created with binary option. When binary format is beneficial to use, allowing the subscription to copy tables in binary in table sync phase reduces the time spent on copy. Discussion: https://postgr.es/m/CAGPVpCQvAziCLknEnygY0v1-KBtg%2BOm-9JHJYZOnNPKFJPompw%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 3 +- doc/src/sgml/logical-replication.sgml | 9 +- doc/src/sgml/ref/alter_subscription.sgml | 4 +- doc/src/sgml/ref/create_subscription.sgml | 21 +- src/backend/catalog/pg_subscription.c | 1 + src/backend/commands/subscriptioncmds.c | 66 ++++- src/backend/replication/logical/tablesync.c | 13 +- src/bin/pg_dump/pg_dump.c | 16 +- src/bin/pg_dump/pg_dump.h | 1 + src/bin/psql/describe.c | 8 +- src/bin/psql/tab-complete.c | 5 +- src/include/catalog/pg_subscription.h | 13 + src/include/commands/subscriptioncmds.h | 1 + src/test/regress/expected/subscription.out | 144 +++++----- src/test/subscription/t/002_types.pl | 291 ++++++++++++++++---- src/test/subscription/t/014_binary.pl | 28 +- 16 files changed, 472 insertions(+), 152 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index c1e4048054..bf8a4c6fd8 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7906,7 +7906,8 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l <structfield>subbinary</structfield> <type>bool</type> </para> <para> - If true, the subscription will request that the publisher send data + If true, the subscription will copy the initial data to synchronize + relations in binary format and request that the publisher send data in binary format </para></entry> </row> diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 1bd5660c87..9540917ea3 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -241,10 +241,11 @@ types of the columns do not need to match, as long as the text representation of the data can be converted to the target type. For example, you can replicate from a column of type <type>integer</type> to a - column of type <type>bigint</type>. The target table can also have - additional columns not provided by the published table. Any such columns - will be filled with the default value as specified in the definition of the - target table. + column of type <type>bigint</type>. Replication in binary format is type + specific and does not allow to replicate data between different types according + to its restrictions. The target table can also have additional columns not provided + by the published table. Any such columns will be filled with the default + value as specified in the definition of the target table. </para> <sect2 id="logical-replication-subscription-slot"> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index ad93553a1d..050155d428 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -213,8 +213,8 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < are <literal>slot_name</literal>, <literal>synchronous_commit</literal>, <literal>binary</literal>, <literal>streaming</literal>, - <literal>disable_on_error</literal>, and - <literal>origin</literal>. + <literal>disable_on_error</literal>, <literal>origin</literal> + and <literal>copy_format</literal>. </para> </listitem> </varlistentry> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index eba72c6af6..9e757ae7de 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -189,8 +189,9 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl <term><literal>binary</literal> (<type>boolean</type>)</term> <listitem> <para> - Specifies whether the subscription will request the publisher to - send the data in binary format (as opposed to text). + Specifies whether the subscription will copy the initial data to + synchronize relations in binary format and also request the publisher + to send the data in binary format too (as opposed to text). The default is <literal>false</literal>. Even when this option is enabled, only data types having binary send and receive functions will be transferred in binary. @@ -349,6 +350,22 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>copy_format</literal> (<type>string</type>)</term> + <listitem> + <para> + Specifies the format in which pre-existing data on the publisher will + copied to the subscriber. Supported formats are + <literal>text</literal> and <literal>binary</literal>. The default is + <literal>text</literal>. + + <literal>binary</literal> format can be selected only if + <literal>binary</literal> is set to <literal>true</literal>. Note + that data will not be copied if <literal>copy_data = false</literal>. + </para> + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index a56ae311c3..da078acefe 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->stream = subform->substream; sub->twophasestate = subform->subtwophasestate; sub->disableonerr = subform->subdisableonerr; + sub->copyformat = subform->subcopyformat; /* Get conninfo */ datum = SysCacheGetAttr(SUBSCRIPTIONOID, diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 464db6d247..0ec45466f5 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -66,6 +66,7 @@ #define SUBOPT_DISABLE_ON_ERR 0x00000400 #define SUBOPT_LSN 0x00000800 #define SUBOPT_ORIGIN 0x00001000 +#define SUBOPT_COPY_FORMAT 0x00002000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -90,6 +91,7 @@ typedef struct SubOpts bool disableonerr; char *origin; XLogRecPtr lsn; + char copy_format; } SubOpts; static List *fetch_table_list(WalReceiverConn *wrconn, List *publications); @@ -146,6 +148,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->disableonerr = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_COPY_FORMAT)) + opts->copy_format = LOGICALREP_COPY_AS_TEXT; /* Parse options */ foreach(lc, stmt_options) @@ -324,6 +328,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->specified_opts |= SUBOPT_LSN; opts->lsn = lsn; } + else if (IsSet(supported_opts, SUBOPT_COPY_FORMAT) && + strcmp(defel->defname, "copy_format") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_COPY_FORMAT)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_COPY_FORMAT; + opts->copy_format = defGetCopyFormat(defel); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -404,6 +417,16 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, "slot_name = NONE", "create_slot = false"))); } } + + if (!opts->binary && + opts->copy_format == LOGICALREP_COPY_AS_BINARY) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s and %s are mutually exclusive options", + "binary = false", "copy_format = binary"))); + } + } /* @@ -560,7 +583,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SLOT_NAME | SUBOPT_COPY_DATA | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | - SUBOPT_DISABLE_ON_ERR | SUBOPT_ORIGIN); + SUBOPT_DISABLE_ON_ERR | SUBOPT_ORIGIN | + SUBOPT_COPY_FORMAT); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -649,6 +673,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subcopyformat - 1] = CharGetDatum(opts.copy_format); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -1054,7 +1079,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, supported_opts = (SUBOPT_SLOT_NAME | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_DISABLE_ON_ERR | - SUBOPT_ORIGIN); + SUBOPT_ORIGIN | SUBOPT_COPY_FORMAT); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); @@ -1118,6 +1143,13 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, replaces[Anum_pg_subscription_suborigin - 1] = true; } + if (IsSet(opts.specified_opts, SUBOPT_COPY_FORMAT)) + { + values[Anum_pg_subscription_subcopyformat - 1] = + CharGetDatum(opts.copy_format); + replaces[Anum_pg_subscription_subcopyformat - 1] = true; + } + update_tuple = true; break; } @@ -2195,3 +2227,33 @@ defGetStreamingMode(DefElem *def) def->defname))); return LOGICALREP_STREAM_OFF; /* keep compiler quiet */ } + +/* + * Extract the copy format value from a DefElem. + */ +char +defGetCopyFormat(DefElem *def) +{ + char *sval; + + /* + * If no parameter value given, set it to text format. + */ + if (!def->arg) + return LOGICALREP_COPY_AS_TEXT; + + /* + * Currently supported formats are "text" and "binary". + */ + sval = defGetString(def); + if (pg_strcasecmp(sval, "text") == 0) + return LOGICALREP_COPY_AS_TEXT; + if (pg_strcasecmp(sval, "binary") == 0) + return LOGICALREP_COPY_AS_BINARY; + + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s value should be either \"text\" or \"binary\"", + def->defname))); + return LOGICALREP_COPY_AS_TEXT; /* keep compiler quiet */ +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 07eea504ba..62c866c9eb 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -101,6 +101,7 @@ #include "catalog/pg_type.h" #include "commands/copy.h" #include "miscadmin.h" +#include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "pgstat.h" #include "replication/logicallauncher.h" @@ -1090,6 +1091,7 @@ copy_table(Relation rel) CopyFromState cstate; List *attnamelist; ParseState *pstate; + List *options = NIL; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1168,6 +1170,15 @@ copy_table(Relation rel) appendStringInfoString(&cmd, ") TO STDOUT"); } + + /* If the publisher is v16 or later, specify the format to copy data. */ + if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 160000) + { + char *format = MySubscription->copyformat == LOGICALREP_COPY_AS_BINARY ? "binary" : "text"; + appendStringInfo(&cmd, " WITH (FORMAT %s)", format); + options = lappend(options, makeDefElem("format", (Node *) makeString(format), -1)); + } + res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL); pfree(cmd.data); if (res->status != WALRCV_OK_COPY_OUT) @@ -1184,7 +1195,7 @@ copy_table(Relation rel) NULL, false, false); attnamelist = make_copy_attnamelist(relmapentry); - cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, NIL); + cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, options); /* Do the copy */ (void) CopyFrom(cstate); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 527c7651ab..2b4e4c52db 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4494,6 +4494,7 @@ getSubscriptions(Archive *fout) int i_subsynccommit; int i_subpublications; int i_subbinary; + int i_subcopyformat; int i, ntups; @@ -4546,9 +4547,14 @@ getSubscriptions(Archive *fout) LOGICALREP_TWOPHASE_STATE_DISABLED); if (fout->remoteVersion >= 160000) - appendPQExpBufferStr(query, " s.suborigin\n"); + appendPQExpBufferStr(query, " s.suborigin,\n"); else - appendPQExpBuffer(query, " '%s' AS suborigin\n", LOGICALREP_ORIGIN_ANY); + appendPQExpBuffer(query, " '%s' AS suborigin,\n", LOGICALREP_ORIGIN_ANY); + + if (fout->remoteVersion >= 160000) + appendPQExpBufferStr(query, " s.subcopyformat\n"); + else + appendPQExpBuffer(query, " '%c' AS subcopyformat\n", LOGICALREP_COPY_AS_TEXT); appendPQExpBufferStr(query, "FROM pg_subscription s\n" @@ -4576,6 +4582,7 @@ getSubscriptions(Archive *fout) i_subtwophasestate = PQfnumber(res, "subtwophasestate"); i_subdisableonerr = PQfnumber(res, "subdisableonerr"); i_suborigin = PQfnumber(res, "suborigin"); + i_subcopyformat = PQfnumber(res, "subcopyformat"); subinfo = pg_malloc(ntups * sizeof(SubscriptionInfo)); @@ -4606,6 +4613,8 @@ getSubscriptions(Archive *fout) subinfo[i].subdisableonerr = pg_strdup(PQgetvalue(res, i, i_subdisableonerr)); subinfo[i].suborigin = pg_strdup(PQgetvalue(res, i, i_suborigin)); + subinfo[i].subcopyformat = + pg_strdup(PQgetvalue(res, i, i_subcopyformat)); /* Decide whether we want to dump it */ selectDumpableObject(&(subinfo[i].dobj), fout); @@ -4687,6 +4696,9 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) if (strcmp(subinfo->subsynccommit, "off") != 0) appendPQExpBuffer(query, ", synchronous_commit = %s", fmtId(subinfo->subsynccommit)); + if (strcmp(subinfo->subcopyformat, "t") != 0) + appendPQExpBuffer(query, ", copy_format = %s", subinfo->subcopyformat); + appendPQExpBufferStr(query, ");\n"); if (subinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index e7cbd8d7ed..f6b7dbcd97 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -662,6 +662,7 @@ typedef struct _SubscriptionInfo char *suborigin; char *subsynccommit; char *subpublications; + char *subcopyformat; } SubscriptionInfo; /* diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index c8a0bb7b3a..b941dc1e88 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6472,7 +6472,7 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false}; if (pset.sversion < 100000) { @@ -6543,6 +6543,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", subskiplsn AS \"%s\"\n", gettext_noop("Skip LSN")); + + /* Copy format is only supported in v16 and higher */ + if (pset.sversion >= 160000) + appendPQExpBuffer(&buf, + ", subcopyformat AS \"%s\"\n", + gettext_noop("Copy Format")); } /* Only display subscriptions in current database. */ diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 5e1882eaea..acc178780c 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1926,7 +1926,7 @@ psql_completion(const char *text, int start, int end) /* ALTER SUBSCRIPTION <name> SET ( */ else if (HeadMatches("ALTER", "SUBSCRIPTION", MatchAny) && TailMatches("SET", "(")) COMPLETE_WITH("binary", "disable_on_error", "origin", "slot_name", - "streaming", "synchronous_commit"); + "streaming", "synchronous_commit", "copy_format"); /* ALTER SUBSCRIPTION <name> SKIP ( */ else if (HeadMatches("ALTER", "SUBSCRIPTION", MatchAny) && TailMatches("SKIP", "(")) COMPLETE_WITH("lsn"); @@ -3269,7 +3269,8 @@ psql_completion(const char *text, int start, int end) else if (HeadMatches("CREATE", "SUBSCRIPTION") && TailMatches("WITH", "(")) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "copy_format"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index b0f2a1705d..fe74cf2cce 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -88,6 +88,7 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subdisableonerr; /* True if a worker error should cause the * subscription to be disabled */ + char subcopyformat BKI_DEFAULT(LOGICALREP_COPY_AS_TEXT); /* Copy format */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -137,6 +138,8 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char copyformat; /* Copy format for subscriptions with binary + * option enabled */ } Subscription; /* Disallow streaming in-progress transactions. */ @@ -154,6 +157,16 @@ typedef struct Subscription */ #define LOGICALREP_STREAM_PARALLEL 'p' +/* + * Copy data as text. This is the default. + */ +#define LOGICALREP_COPY_AS_TEXT 't' + +/* + * Copy data as binary if the binary options is enabled in the subscription. + */ +#define LOGICALREP_COPY_AS_BINARY 'b' + extern Subscription *GetSubscription(Oid subid, bool missing_ok); extern void FreeSubscription(Subscription *sub); extern void DisableSubscription(Oid subid); diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..7937defea5 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -27,5 +27,6 @@ extern ObjectAddress AlterSubscriptionOwner(const char *name, Oid newOwnerId); extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern char defGetCopyFormat(DefElem *def); #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 3f99b14394..cf8bdec5dc 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -114,18 +114,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -143,10 +143,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -163,10 +163,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/12345 | t (1 row) -- ok - with lsn = NONE @@ -175,10 +175,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/0 | t (1 row) BEGIN; @@ -210,10 +210,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | local | dbname=regress_doesnotexist2 | 0/0 | t (1 row) -- rename back to keep the rest simple @@ -247,19 +247,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub; @@ -271,27 +271,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) -- fail - publication already exists @@ -306,10 +306,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) -- fail - publication used more than once @@ -324,10 +324,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub; @@ -363,10 +363,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) --fail - alter of two_phase option not supported. @@ -375,10 +375,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -388,10 +388,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -404,18 +404,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index 6b5853b80b..1be2f3c304 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -19,6 +19,11 @@ my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->start; +# Create binary subscriber node +my $node_subscriber_binary = PostgreSQL::Test::Cluster->new('subscriber_binary'); +$node_subscriber_binary->init(allows_streaming => 'logical'); +$node_subscriber_binary->start; + # Create some preexisting content on publisher my $ddl = qq( CREATE EXTENSION hstore WITH SCHEMA public; @@ -104,6 +109,100 @@ my $ddl = qq( # Setup structure on both nodes $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +# Insert initial test data +$node_publisher->safe_psql( + 'postgres', qq( + -- test_tbl_one_array_col + INSERT INTO tst_one_array (a, b) VALUES + (1, '{1, 2, 3}'), + (2, '{2, 3, 1}'); + + -- test_tbl_arrays + INSERT INTO tst_arrays (a, b, c, d) VALUES + ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), + ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'); + + -- test_tbl_single_enum + INSERT INTO tst_one_enum (a, b) VALUES + (1, 'a'), + (2, 'b'); + + -- test_tbl_enums + INSERT INTO tst_enums (a, b) VALUES + ('a', '{b, c}'), + ('b', '{c, a}'); + + -- test_tbl_single_composites + INSERT INTO tst_one_comp (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composites + INSERT INTO tst_comps (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]); + + -- test_tbl_composite_with_enums + INSERT INTO tst_comp_enum (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composite_with_enums_array + INSERT INTO tst_comp_enum_array (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]); + + -- test_tbl_composite_with_single_enums_array_in_composite + INSERT INTO tst_comp_one_enum_array (a, b) VALUES + (1, ROW(1.0, '{a, b, c}', 1)), + (2, ROW(2.0, '{a, b, c}', 2)); + + -- test_tbl_composite_with_enums_array_in_composite + INSERT INTO tst_comp_enum_what (a, b) VALUES + (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), + (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]); + + -- test_tbl_mixed_composites + INSERT INTO tst_comp_mix_array (a, b) VALUES + (ROW( + ROW(1,'a',1), + ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], + 'a', + '{a,b,NULL,c}'), + ARRAY[ + ROW( + ROW(1,'a',1), + ARRAY[ + ROW(1,'a',1)::tst_comp_basic_t, + ROW(2,'b',2)::tst_comp_basic_t, + NULL + ], + 'a', + '{a,b,c}' + )::tst_comp_mix_t + ] + ); + + -- test_tbl_range + INSERT INTO tst_range (a, b) VALUES + (1, '[1, 10]'), + (2, '[2, 20]'); + + -- test_tbl_range_array + INSERT INTO tst_range_array (a, b, c) VALUES + (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), + (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'); + + -- tst_hstore + INSERT INTO tst_hstore (a, b) VALUES + (1, '"a"=>"1"'), + (2, '"zzz"=>"foo"'); + + -- tst_dom_constr + INSERT INTO tst_dom_constr VALUES (10); +)); # Setup logical replication my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; @@ -113,89 +212,126 @@ $node_publisher->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_slot)" ); +$node_subscriber_binary->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_binary CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_binary_slot, binary = true, copy_format = 'binary')" +); # Wait for initial sync to finish as well $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); +$node_subscriber_binary->wait_for_subscription_sync($node_publisher, 'tap_sub_binary'); -# Insert initial test data +my $sync_check = qq( + SET timezone = '+2'; + SELECT a, b FROM tst_one_array ORDER BY a; + SELECT a, b, c, d FROM tst_arrays ORDER BY a; + SELECT a, b FROM tst_one_enum ORDER BY a; + SELECT a, b FROM tst_enums ORDER BY a; + SELECT a, b FROM tst_one_comp ORDER BY a; + SELECT a, b FROM tst_comps ORDER BY a; + SELECT a, b FROM tst_comp_enum ORDER BY a; + SELECT a, b FROM tst_comp_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_one_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_enum_what ORDER BY a; + SELECT a, b FROM tst_comp_mix_array ORDER BY a; + SELECT a, b FROM tst_range ORDER BY a; + SELECT a, b, c FROM tst_range_array ORDER BY a; + SELECT a, b FROM tst_hstore ORDER BY a; +); + +# Check the synced data on subscribers +my $result = $node_subscriber->safe_psql('postgres', $sync_check); +my $result_binary = $node_subscriber_binary->safe_psql('postgres', $sync_check); + +my $sync_result = '1|{1,2,3} +2|{2,3,1} +{1,2,3}|{a,b,c}|{1.1,2.2,3.3}|{"1 day","2 days","3 days"} +{2,3,1}|{b,c,a}|{2.2,3.3,1.1}|{00:02:00,00:03:00,00:01:00} +1|a +2|b +a|{b,c} +b|{c,a} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,"{a,b,c}",1) +2|(2,"{a,b,c}",2) +(1,"{a,b,c}",1)|{"(1,\"{a,b,c}\",1)"} +(2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} +("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +1|[1,11) +2|[2,21) +1|["2014-08-04 00:00:00+02",infinity)|{"[1,3)","[10,21)"} +2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} +1|"a"=>"1" +2|"zzz"=>"foo"'; + +is( $result, $sync_result, 'check initial sync on subscriber'); +is( $result_binary, $sync_result, 'check initial sync on subscriber in binary'); + +# Insert test data for update to apply changes $node_publisher->safe_psql( 'postgres', qq( -- test_tbl_one_array_col INSERT INTO tst_one_array (a, b) VALUES - (1, '{1, 2, 3}'), - (2, '{2, 3, 1}'), (3, '{3, 2, 1}'), (4, '{4, 3, 2}'), (5, '{5, NULL, 3}'); -- test_tbl_arrays INSERT INTO tst_arrays (a, b, c, d) VALUES - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'), ('{3, 1, 2}', '{"c", "a", "b"}', '{3.3, 1.1, 2.2}', '{"3 years", "1 year", "2 years"}'), ('{4, 1, 2}', '{"d", "a", "b"}', '{4.4, 1.1, 2.2}', '{"4 years", "1 year", "2 years"}'), ('{5, NULL, NULL}', '{"e", NULL, "b"}', '{5.5, 1.1, NULL}', '{"5 years", NULL, NULL}'); -- test_tbl_single_enum INSERT INTO tst_one_enum (a, b) VALUES - (1, 'a'), - (2, 'b'), (3, 'c'), (4, 'd'), (5, NULL); -- test_tbl_enums INSERT INTO tst_enums (a, b) VALUES - ('a', '{b, c}'), - ('b', '{c, a}'), ('c', '{b, a}'), ('d', '{c, b}'), ('e', '{d, NULL}'); -- test_tbl_single_composites INSERT INTO tst_one_comp (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, NULL, 5)); -- test_tbl_composites INSERT INTO tst_comps (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_basic_t]), (ROW(4.0, 'd', 4), ARRAY[ROW(4, 'd', 3)::tst_comp_basic_t]), (ROW(5.0, 'e', NULL), ARRAY[NULL, ROW(5, NULL, 5)::tst_comp_basic_t]); -- test_tbl_composite_with_enums INSERT INTO tst_comp_enum (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, 'e', NULL)); -- test_tbl_composite_with_enums_array INSERT INTO tst_comp_enum_array (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_enum_t]), (ROW(4.0, 'd', 3), ARRAY[ROW(3, 'd', 3)::tst_comp_enum_t]), (ROW(5.0, 'e', 3), ARRAY[ROW(3, 'e', 3)::tst_comp_enum_t, NULL]); -- test_tbl_composite_with_single_enums_array_in_composite INSERT INTO tst_comp_one_enum_array (a, b) VALUES - (1, ROW(1.0, '{a, b, c}', 1)), - (2, ROW(2.0, '{a, b, c}', 2)), (3, ROW(3.0, '{a, b, c}', 3)), (4, ROW(4.0, '{c, b, d}', 4)), (5, ROW(5.0, '{NULL, e, NULL}', 5)); -- test_tbl_composite_with_enums_array_in_composite INSERT INTO tst_comp_enum_what (a, b) VALUES - (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), - (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]), (ROW(3.0, '{c, a, b}', 1), ARRAY[ROW(3, '{c, a, b}', 1)::tst_comp_enum_array_t]), (ROW(4.0, '{c, b, d}', 4), ARRAY[ROW(4, '{c, b, d}', 4)::tst_comp_enum_array_t]), (ROW(5.0, '{c, NULL, b}', NULL), ARRAY[ROW(5, '{c, e, b}', 1)::tst_comp_enum_array_t]); @@ -203,10 +339,10 @@ $node_publisher->safe_psql( -- test_tbl_mixed_composites INSERT INTO tst_comp_mix_array (a, b) VALUES (ROW( - ROW(1,'a',1), - ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], - 'a', - '{a,b,NULL,c}'), + ROW(2,'b',2), + ARRAY[ROW(2,'b',2)::tst_comp_basic_t, ROW(3,'c',3)::tst_comp_basic_t], + 'b', + '{b,c,NULL,d}'), ARRAY[ ROW( ROW(1,'a',1), @@ -223,36 +359,29 @@ $node_publisher->safe_psql( -- test_tbl_range INSERT INTO tst_range (a, b) VALUES - (1, '[1, 10]'), - (2, '[2, 20]'), (3, '[3, 30]'), (4, '[4, 40]'), (5, '[5, 50]'); -- test_tbl_range_array INSERT INTO tst_range_array (a, b, c) VALUES - (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), - (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'), (3, tstzrange('Fri Aug 01 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[3,4]"}'), (4, tstzrange('Thu Jul 31 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[4,5]", NULL, "[40,50]"}'), (5, NULL, NULL); -- tst_hstore INSERT INTO tst_hstore (a, b) VALUES - (1, '"a"=>"1"'), - (2, '"zzz"=>"foo"'), (3, '"123"=>"321"'), (4, '"yellow horse"=>"moaned"'); -- tst_dom_constr - INSERT INTO tst_dom_constr VALUES (10); + INSERT INTO tst_dom_constr VALUES (11); )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -my $result = $node_subscriber->safe_psql( - 'postgres', qq( +my $initial_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -268,9 +397,13 @@ my $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $initial_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $initial_check); -is( $result, '1|{1,2,3} +my $initial_result = '1|{1,2,3} 2|{2,3,1} 3|{3,2,1} 4|{4,3,2} @@ -321,6 +454,7 @@ e|{d,NULL} (4,"{c,b,d}",4)|{"(4,\"{c,b,d}\",4)"} (5,"{c,NULL,b}",)|{"(5,\"{c,e,b}\",1)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[1,11) 2|[2,21) 3|[3,31) @@ -334,8 +468,10 @@ e|{d,NULL} 1|"a"=>"1" 2|"zzz"=>"foo" 3|"123"=>"321" -4|"yellow horse"=>"moaned"', - 'check replicated inserts on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $initial_result, 'check replicated inserts on subscriber'); +is( $result_binary, $initial_result, 'check replicated inserts on subscriber in binary'); # Run batch of updates $node_publisher->safe_psql( @@ -370,10 +506,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $update_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -389,9 +524,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $update_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $update_check); -is( $result, '1|{4,5,6} +my $update_result = '1|{4,5,6} 2|{2,3,1} 3|{3,2,1} 4|{4,5,6,1} @@ -442,6 +581,7 @@ e|{e,d} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")",NULL} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[100,1001) 2|[2,21) 3|[3,31) @@ -455,8 +595,10 @@ e|{e,d} 1|"updated"=>"value" 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated updates on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $update_result, 'check replicated updates on subscriber'); +is( $result_binary, $update_result, 'check replicated updates on subscriber in binary'); # Run batch of deletes $node_publisher->safe_psql( @@ -490,10 +632,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $delete_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -509,9 +650,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $delete_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $delete_check); -is( $result, '3|{3,2,1} +my $delete_result = '3|{3,2,1} 4|{4,5,6,1} 5|{4,5,6,1} {3,1,2}|{c,a,b}|{3.3,1.1,2.2}|{"3 years","1 year","2 years"} @@ -540,26 +685,56 @@ e|{e,d} (2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} 3|["2014-08-01 00:00:00+02","2014-08-04 00:00:00+02")|{"[3,5)"} 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated deletes on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $delete_result, 'check replicated deletes on subscriber'); +is( $result_binary, $delete_result, 'check replicated deletes on subscriber in binary'); # Test a domain with a constraint backed by a SQL-language function, # which needs an active snapshot in order to operate. $node_publisher->safe_psql('postgres', - "INSERT INTO tst_dom_constr VALUES (11)"); + "INSERT INTO tst_dom_constr VALUES (12)"); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); + +my $domain_check = qq(SELECT sum(a) FROM tst_dom_constr); + +$result = $node_subscriber->safe_psql('postgres', $domain_check); +$result_binary = $node_subscriber->safe_psql('postgres', $domain_check); -$result = - $node_subscriber->safe_psql('postgres', - "SELECT sum(a) FROM tst_dom_constr"); -is($result, '21', 'sql-function constraint on domain'); +is( $result, '33', 'sql-function constraint on domain'); +is( $result_binary, '33', 'sql-function constraint on domain'); + +# Test whether subscriber fails in case of column type mismatch +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a int); + INSERT INTO tst_binary_mismatch VALUES (1); +); +$node_publisher->safe_psql('postgres', $ddl); + +$ddl = qq( + CREATE TABLE tst_binary_mismatch (a bigint); +); +$node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION'); +$node_subscriber_binary->safe_psql('postgres', 'ALTER SUBSCRIPTION tap_sub_binary REFRESH PUBLICATION'); + +# Binary enabled subscription should fail +$node_subscriber_binary->wait_for_log(qr/ERROR: ( [A-Z0-9]+:)? insufficient data left in message/); + +# Binary disabled subscription should succeed +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); $node_subscriber->stop('fast'); +$node_subscriber_binary->stop('fast'); $node_publisher->stop('fast'); done_testing(); diff --git a/src/test/subscription/t/014_binary.pl b/src/test/subscription/t/014_binary.pl index e53e23da3e..04ad6de08f 100644 --- a/src/test/subscription/t/014_binary.pl +++ b/src/test/subscription/t/014_binary.pl @@ -36,6 +36,16 @@ my $ddl = qq( $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +# Insert some content and make sure it's synced to subscriber +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO public.test_arrays (a, b, c) VALUES + ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'); + + INSERT INTO public.test_numerical (a, b, c, d) VALUES + (1, 1.2, 1.3, 10); + )); + # Configure logical replication $node_publisher->safe_psql('postgres', "CREATE PUBLICATION tpub FOR ALL TABLES"); @@ -48,27 +58,35 @@ $node_subscriber->safe_psql('postgres', # Ensure nodes are in sync with each other $node_subscriber->wait_for_subscription_sync($node_publisher, 'tsub'); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); + +is( $result, '1|1.2|1.3|10 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three}', 'check syned data on subscriber'); + # Insert some content and make sure it's replicated across $node_publisher->safe_psql( 'postgres', qq( INSERT INTO public.test_arrays (a, b, c) VALUES - ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'), ('{3,1,2}', '{1.3, 1.1, 1.2}', '{"three", "one", "two"}'); INSERT INTO public.test_numerical (a, b, c, d) VALUES - (1, 1.2, 1.3, 10), (2, 2.2, 2.3, 20), (3, 3.2, 3.3, 30); )); $node_publisher->wait_for_catchup('tsub'); -my $result = $node_subscriber->safe_psql('postgres', - "SELECT a, b, c, d FROM test_numerical ORDER BY a"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); is( $result, '1|1.2|1.3|10 2|2.2|2.3|20 -3|3.2|3.3|30', 'check replicated data on subscriber'); +3|3.2|3.3|30 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three} +{3,1,2}|{1.3,1.1,1.2}|{three,one,two}', 'check replicated data on subscriber'); # Test updates as well $node_publisher->safe_psql( -- 2.25.1 ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-01-30 12:34 ` Bharath Rupireddy <[email protected]> 2023-02-16 09:18 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2 siblings, 1 reply; 33+ messages in thread From: Bharath Rupireddy @ 2023-01-30 12:34 UTC (permalink / raw) To: Melih Mutlu <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers On Mon, Jan 30, 2023 at 4:19 PM Melih Mutlu <[email protected]> wrote: > Thanks for providing an updated patch. >> On Thu, Jan 12, 2023 at 1:53 PM Melih Mutlu <[email protected]> wrote: >> 1. The performance numbers posted upthread [1] look impressive for the >> use-case tried, that's a 2.25X improvement or 55.6% reduction in >> execution times. However, it'll be great to run a few more varied >> tests to confirm the benefit. > > Sure, do you have any specific test case or suggestion in mind? Have a huge amount of publisher's table (with mix of columns like int, text, double, bytea and so on) prior data so that the subscriber's table sync workers have to do a "good" amount of work to copy, then measure the copy_table time with and without patch. >> 2. It'll be great to capture the perf report to see if the time spent >> in copy_table() is reduced with the patch. > > Will provide that in another email soon. Thanks. >> 4. Why to add tests to existing 002_types.pl? Can we add a new file >> with all the data types covered? > > Since 002_types.pl is where the replication of data types are covered. I thought it would be okay to test replication with the binary option in that file. > Sure, we can add a new file with different data types for testing subscriptions with binary option. But then I feel like it would have too many duplicated lines with 002_types.pl. > If you think that 002_types.pl lacks some data types needs to be tested, then we should add those into 002_types.pl too whether we test subscription with binary option in that file or some other place, right? > > Previously, it wasn't actually testing the initial table sync since all tables were empty when subscription was created. > I just simply split the data initially inserted to test initial table sync. > > With this patch, it inserts the first two rows for all data types before subscriptions get created. > You can see these lines: It'd be better and clearer to have a separate TAP test file IMO since the focus of the feature here isn't to just test for data types. With separate tests, you can verify "ERROR: incorrect binary data format in logical replication column 1" cases. >> 8. Note that the COPY binary format isn't portable across platforms >> (Windows to Linux for instance) or major versions >> https://www.postgresql.org/docs/devel/sql-copy.html, whereas, logical >> replication is https://www.postgresql.org/docs/devel/logical-replication.html. >> I don't see any handling of such cases in copy_table but only a check >> for the publisher version. I think we need to account for all the >> cases - allow binary COPY only when publisher and subscriber are of >> same versions, architectures, platforms. The trick here is how we >> identify if the subscriber is of the same type and taste >> (architectures and platforms) as the publisher. Looking for >> PG_VERSION_STR of publisher and subscriber might be naive, but I'm not >> sure if there's a better way to do it. > > I think having the "copy_format" option with text as default, like I replied to your 3rd review above, will keep things as they are now. > The patch now will only allow users to choose binary copy as well, if they want it and acknowledge the limitations that come with binary copy. > COPY command's portability issues shouldn't be an issue right now, since the patch still supports text format. Right? With the above said, do you think checking for publisher versions is needed? The user can decide to enable/disable binary COPY based on the publisher's version no? + /* If the publisher is v16 or later, specify the format to copy data. */ + if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 160000) + { Few more comments on v7: 1. + Specifies the format in which pre-existing data on the publisher will + copied to the subscriber. Supported formats are + <literal>text</literal> and <literal>binary</literal>. The default is + <literal>text</literal>. It'll be good to call out the cases in the documentation as to where copy_format can be enabled and needs to be disabled. 2. + errmsg("%s value should be either \"text\" or \"binary\"", How about "value must be either ...."? 3. + if (!opts->binary && + opts->copy_format == LOGICALREP_COPY_AS_BINARY) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s and %s are mutually exclusive options", + "binary = false", "copy_format = binary"))); + "CREATE SUBSCRIPTION tap_sub_binary CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_binary_slot, binary = true, copy_format = 'binary')" Why should the subscription's binary option and copy_format option be tied at all? Tying these two options hurts usability. Is there a fundamental reason? I think they both are/must be independent. One deals with data types and another deals with how initial table data is copied. -- Bharath Rupireddy PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-30 12:34 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> @ 2023-02-16 09:18 ` Melih Mutlu <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Melih Mutlu @ 2023-02-16 09:18 UTC (permalink / raw) To: Bharath Rupireddy <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi, Please see the attached patch for following changes. Bharath Rupireddy <[email protected]>, 30 Oca 2023 Pzt, 15:34 tarihinde şunu yazdı: > On Mon, Jan 30, 2023 at 4:19 PM Melih Mutlu <[email protected]> > wrote: It'd be better and clearer to have a separate TAP test file IMO since > the focus of the feature here isn't to just test for data types. With > separate tests, you can verify "ERROR: incorrect binary data format > in logical replication column 1" cases. > Moved some tests from 002_types.pl to 014_binary.pl since this is where most binary features are tested. It covers now "incorrect data format" cases too. Also added some regression tests for copy_format parameter. > With the above said, do you think checking for publisher versions is > needed? The user can decide to enable/disable binary COPY based on the > publisher's version no? > + /* If the publisher is v16 or later, specify the format to copy data. > */ > + if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 160000) > + { > If the user decides to enable it, then it might be nice to not allow it for previous versions. But I'm not sure. I'm okay to remove it if you all agree. > Few more comments on v7: > 1. > + Specifies the format in which pre-existing data on the > publisher will > + copied to the subscriber. Supported formats are > + <literal>text</literal> and <literal>binary</literal>. The > default is > + <literal>text</literal>. > It'll be good to call out the cases in the documentation as to where > copy_format can be enabled and needs to be disabled. > Modified that description a bit. Can you check if that's okay now? > 2. > + errmsg("%s value should be either \"text\" or \"binary\"", > How about "value must be either ...."? > Done > 3. > Why should the subscription's binary option and copy_format option be > tied at all? Tying these two options hurts usability. Is there a > fundamental reason? I think they both are/must be independent. One > deals with data types and another deals with how initial table data is > copied. > My initial purpose with this patch was just making subscriptions with binary option enabled fully binary from initial copy to apply. Things have changed a bit when we decided to move binary copy behind a parameter. I didn't actually think there would be any use case where a user wants the initial copy to be in binary format for a sub with binary = false. Do you think it would be useful to copy in binary even for a sub with binary disabled? Thanks, -- Melih Mutlu Microsoft Attachments: [application/octet-stream] v8-0001-Allow-logical-replication-to-copy-table-in-binary.patch (78.0K, ../../CAGPVpCQYi9AYQSS=RmGgVNjz5ZEnLB8mACwd9aioVhLmbgiAMA@mail.gmail.com/3-v8-0001-Allow-logical-replication-to-copy-table-in-binary.patch) download | inline diff: From 38490cd763850d8aee9e0efda597fe61711527be Mon Sep 17 00:00:00 2001 From: Melih Mutlu <[email protected]> Date: Mon, 8 Aug 2022 14:14:44 +0300 Subject: [PATCH v8] Allow logical replication to copy table in binary If binary option is enabled in a subscription and copy_format is set to binary, then copy tables in binary format during table synchronization. The patch introduces a new parameter, copy_format, to CREATE SUBSCRIPTION to allow to choose the format used in initial table synchronization. Without this patch, table are copied in text format even if the subscription is created with binary option. When binary format is beneficial to use, allowing the subscription to copy tables in binary in table sync phase may reduce the time spent on copy depending on column types. Discussion: https://postgr.es/m/CAGPVpCQvAziCLknEnygY0v1-KBtg%2BOm-9JHJYZOnNPKFJPompw%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 12 +- doc/src/sgml/logical-replication.sgml | 9 +- doc/src/sgml/ref/alter_subscription.sgml | 4 +- doc/src/sgml/ref/create_subscription.sgml | 17 ++ src/backend/catalog/pg_subscription.c | 1 + src/backend/commands/subscriptioncmds.c | 84 +++++- src/backend/replication/logical/tablesync.c | 13 +- src/bin/pg_dump/pg_dump.c | 17 +- src/bin/pg_dump/pg_dump.h | 1 + src/bin/psql/describe.c | 8 +- src/bin/psql/tab-complete.c | 5 +- src/include/catalog/pg_subscription.h | 13 + src/include/commands/subscriptioncmds.h | 1 + src/test/regress/expected/subscription.out | 192 ++++++++------ src/test/regress/sql/subscription.sql | 26 ++ src/test/subscription/t/002_types.pl | 269 +++++++++++++++----- src/test/subscription/t/014_binary.pl | 82 +++++- 17 files changed, 604 insertions(+), 150 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index c1e4048054..14e8cad531 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7907,7 +7907,6 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para> <para> If true, the subscription will request that the publisher send data - in binary format </para></entry> </row> @@ -8002,6 +8001,17 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l origin. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subcopyformat</structfield> <type>bool</type> + </para> + <para> + Specifies the format for copying the initial data in a subscription. + <literal>t</literal> indicates text, <literal>b</literal> indicates + binary format. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 1bd5660c87..9540917ea3 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -241,10 +241,11 @@ types of the columns do not need to match, as long as the text representation of the data can be converted to the target type. For example, you can replicate from a column of type <type>integer</type> to a - column of type <type>bigint</type>. The target table can also have - additional columns not provided by the published table. Any such columns - will be filled with the default value as specified in the definition of the - target table. + column of type <type>bigint</type>. Replication in binary format is type + specific and does not allow to replicate data between different types according + to its restrictions. The target table can also have additional columns not provided + by the published table. Any such columns will be filled with the default + value as specified in the definition of the target table. </para> <sect2 id="logical-replication-subscription-slot"> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 964fcbb8ff..c560d71f91 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -213,8 +213,8 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < are <literal>slot_name</literal>, <literal>synchronous_commit</literal>, <literal>binary</literal>, <literal>streaming</literal>, - <literal>disable_on_error</literal>, and - <literal>origin</literal>. + <literal>disable_on_error</literal>, <literal>origin</literal> + and <literal>copy_format</literal>. </para> </listitem> </varlistentry> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 51c45f17c7..f01b762d81 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -349,6 +349,23 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>copy_format</literal> (<type>string</type>)</term> + <listitem> + <para> + Specifies the format in which pre-existing data on the publisher will + copied to the subscriber. Supported formats are + <literal>text</literal> and <literal>binary</literal>. The default is + <literal>text</literal>. + <literal>binary</literal> format can be selected only if + <literal>binary</literal> is set to <literal>true</literal>. Also, + the binary format is very data type specific, it will not allow copying + between different column types as opposed to text format. Note that + data will not be copied if <literal>copy_data = false</literal>. + </para> + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index a56ae311c3..da078acefe 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -71,6 +71,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->stream = subform->substream; sub->twophasestate = subform->subtwophasestate; sub->disableonerr = subform->subdisableonerr; + sub->copyformat = subform->subcopyformat; /* Get conninfo */ datum = SysCacheGetAttr(SUBSCRIPTIONOID, diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 464db6d247..4e4023623d 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -66,6 +66,7 @@ #define SUBOPT_DISABLE_ON_ERR 0x00000400 #define SUBOPT_LSN 0x00000800 #define SUBOPT_ORIGIN 0x00001000 +#define SUBOPT_COPY_FORMAT 0x00002000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -90,6 +91,7 @@ typedef struct SubOpts bool disableonerr; char *origin; XLogRecPtr lsn; + char copy_format; } SubOpts; static List *fetch_table_list(WalReceiverConn *wrconn, List *publications); @@ -146,6 +148,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->disableonerr = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_COPY_FORMAT)) + opts->copy_format = LOGICALREP_COPY_AS_TEXT; /* Parse options */ foreach(lc, stmt_options) @@ -324,6 +328,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->specified_opts |= SUBOPT_LSN; opts->lsn = lsn; } + else if (IsSet(supported_opts, SUBOPT_COPY_FORMAT) && + strcmp(defel->defname, "copy_format") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_COPY_FORMAT)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_COPY_FORMAT; + opts->copy_format = defGetCopyFormat(defel); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -404,6 +417,17 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, "slot_name = NONE", "create_slot = false"))); } } + + if (IsSet(opts->specified_opts, SUBOPT_BINARY) && + !opts->binary && + opts->copy_format == LOGICALREP_COPY_AS_BINARY) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s and %s are mutually exclusive options", + "binary = false", "copy_format = binary"))); + } + } /* @@ -560,7 +584,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SLOT_NAME | SUBOPT_COPY_DATA | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | - SUBOPT_DISABLE_ON_ERR | SUBOPT_ORIGIN); + SUBOPT_DISABLE_ON_ERR | SUBOPT_ORIGIN | + SUBOPT_COPY_FORMAT); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -649,6 +674,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subcopyformat - 1] = CharGetDatum(opts.copy_format); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -1054,7 +1080,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, supported_opts = (SUBOPT_SLOT_NAME | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_DISABLE_ON_ERR | - SUBOPT_ORIGIN); + SUBOPT_ORIGIN | SUBOPT_COPY_FORMAT); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); @@ -1091,6 +1117,15 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, if (IsSet(opts.specified_opts, SUBOPT_BINARY)) { + /* Do not allow binary = false with copy_format = binary */ + if (!opts.binary && + sub->copyformat == LOGICALREP_COPY_AS_BINARY && + !IsSet(opts.specified_opts, SUBOPT_COPY_FORMAT)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot set %s for a subscription with %s", + "binary = false", "copy_format = binary"))); + values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary); replaces[Anum_pg_subscription_subbinary - 1] = true; @@ -1118,6 +1153,21 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, replaces[Anum_pg_subscription_suborigin - 1] = true; } + if (IsSet(opts.specified_opts, SUBOPT_COPY_FORMAT)) + { + /* Do not allow binary = false with copy_format = binary */ + if (opts.copy_format == LOGICALREP_COPY_AS_BINARY && + !sub->binary && + !IsSet(opts.specified_opts, SUBOPT_BINARY)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot set %s for a subscription with %s", + "copy_format = binary", "binary = false"))); + values[Anum_pg_subscription_subcopyformat - 1] = + CharGetDatum(opts.copy_format); + replaces[Anum_pg_subscription_subcopyformat - 1] = true; + } + update_tuple = true; break; } @@ -2195,3 +2245,33 @@ defGetStreamingMode(DefElem *def) def->defname))); return LOGICALREP_STREAM_OFF; /* keep compiler quiet */ } + +/* + * Extract the copy format value from a DefElem. + */ +char +defGetCopyFormat(DefElem *def) +{ + char *sval; + + /* + * If no parameter value given, set it to text format. + */ + if (!def->arg) + return LOGICALREP_COPY_AS_TEXT; + + /* + * Currently supported formats are "text" and "binary". + */ + sval = defGetString(def); + if (pg_strcasecmp(sval, "text") == 0) + return LOGICALREP_COPY_AS_TEXT; + if (pg_strcasecmp(sval, "binary") == 0) + return LOGICALREP_COPY_AS_BINARY; + + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s value must be either \"text\" or \"binary\"", + def->defname))); + return LOGICALREP_COPY_AS_TEXT; /* keep compiler quiet */ +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 07eea504ba..62c866c9eb 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -101,6 +101,7 @@ #include "catalog/pg_type.h" #include "commands/copy.h" #include "miscadmin.h" +#include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "pgstat.h" #include "replication/logicallauncher.h" @@ -1090,6 +1091,7 @@ copy_table(Relation rel) CopyFromState cstate; List *attnamelist; ParseState *pstate; + List *options = NIL; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1168,6 +1170,15 @@ copy_table(Relation rel) appendStringInfoString(&cmd, ") TO STDOUT"); } + + /* If the publisher is v16 or later, specify the format to copy data. */ + if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 160000) + { + char *format = MySubscription->copyformat == LOGICALREP_COPY_AS_BINARY ? "binary" : "text"; + appendStringInfo(&cmd, " WITH (FORMAT %s)", format); + options = lappend(options, makeDefElem("format", (Node *) makeString(format), -1)); + } + res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL); pfree(cmd.data); if (res->status != WALRCV_OK_COPY_OUT) @@ -1184,7 +1195,7 @@ copy_table(Relation rel) NULL, false, false); attnamelist = make_copy_attnamelist(relmapentry); - cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, NIL); + cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, options); /* Do the copy */ (void) CopyFrom(cstate); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 527c7651ab..9b97aa06e5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4494,6 +4494,7 @@ getSubscriptions(Archive *fout) int i_subsynccommit; int i_subpublications; int i_subbinary; + int i_subcopyformat; int i, ntups; @@ -4546,9 +4547,15 @@ getSubscriptions(Archive *fout) LOGICALREP_TWOPHASE_STATE_DISABLED); if (fout->remoteVersion >= 160000) - appendPQExpBufferStr(query, " s.suborigin\n"); + { + appendPQExpBufferStr(query, " s.suborigin,\n"); + appendPQExpBufferStr(query, " s.subcopyformat\n"); + } else - appendPQExpBuffer(query, " '%s' AS suborigin\n", LOGICALREP_ORIGIN_ANY); + { + appendPQExpBuffer(query, " '%s' AS suborigin,\n", LOGICALREP_ORIGIN_ANY); + appendPQExpBuffer(query, " '%c' AS subcopyformat\n", LOGICALREP_COPY_AS_TEXT); + } appendPQExpBufferStr(query, "FROM pg_subscription s\n" @@ -4576,6 +4583,7 @@ getSubscriptions(Archive *fout) i_subtwophasestate = PQfnumber(res, "subtwophasestate"); i_subdisableonerr = PQfnumber(res, "subdisableonerr"); i_suborigin = PQfnumber(res, "suborigin"); + i_subcopyformat = PQfnumber(res, "subcopyformat"); subinfo = pg_malloc(ntups * sizeof(SubscriptionInfo)); @@ -4606,6 +4614,8 @@ getSubscriptions(Archive *fout) subinfo[i].subdisableonerr = pg_strdup(PQgetvalue(res, i, i_subdisableonerr)); subinfo[i].suborigin = pg_strdup(PQgetvalue(res, i, i_suborigin)); + subinfo[i].subcopyformat = + pg_strdup(PQgetvalue(res, i, i_subcopyformat)); /* Decide whether we want to dump it */ selectDumpableObject(&(subinfo[i].dobj), fout); @@ -4687,6 +4697,9 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) if (strcmp(subinfo->subsynccommit, "off") != 0) appendPQExpBuffer(query, ", synchronous_commit = %s", fmtId(subinfo->subsynccommit)); + if (strcmp(subinfo->subcopyformat, "t") != 0) + appendPQExpBuffer(query, ", copy_format = %s", subinfo->subcopyformat); + appendPQExpBufferStr(query, ");\n"); if (subinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index e7cbd8d7ed..f6b7dbcd97 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -662,6 +662,7 @@ typedef struct _SubscriptionInfo char *suborigin; char *subsynccommit; char *subpublications; + char *subcopyformat; } SubscriptionInfo; /* diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index c8a0bb7b3a..8e9523aba4 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6472,7 +6472,7 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false}; if (pset.sversion < 100000) { @@ -6528,9 +6528,15 @@ describeSubscriptions(const char *pattern, bool verbose) gettext_noop("Disable on error")); if (pset.sversion >= 160000) + { appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n", gettext_noop("Origin")); + /* Copy format is only supported in v16 and higher */ + appendPQExpBuffer(&buf, + ", subcopyformat AS \"%s\"\n", + gettext_noop("Copy Format")); + } appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 5e1882eaea..acc178780c 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1926,7 +1926,7 @@ psql_completion(const char *text, int start, int end) /* ALTER SUBSCRIPTION <name> SET ( */ else if (HeadMatches("ALTER", "SUBSCRIPTION", MatchAny) && TailMatches("SET", "(")) COMPLETE_WITH("binary", "disable_on_error", "origin", "slot_name", - "streaming", "synchronous_commit"); + "streaming", "synchronous_commit", "copy_format"); /* ALTER SUBSCRIPTION <name> SKIP ( */ else if (HeadMatches("ALTER", "SUBSCRIPTION", MatchAny) && TailMatches("SKIP", "(")) COMPLETE_WITH("lsn"); @@ -3269,7 +3269,8 @@ psql_completion(const char *text, int start, int end) else if (HeadMatches("CREATE", "SUBSCRIPTION") && TailMatches("WITH", "(")) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "copy_format"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index b0f2a1705d..fe74cf2cce 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -88,6 +88,7 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subdisableonerr; /* True if a worker error should cause the * subscription to be disabled */ + char subcopyformat BKI_DEFAULT(LOGICALREP_COPY_AS_TEXT); /* Copy format */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -137,6 +138,8 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char copyformat; /* Copy format for subscriptions with binary + * option enabled */ } Subscription; /* Disallow streaming in-progress transactions. */ @@ -154,6 +157,16 @@ typedef struct Subscription */ #define LOGICALREP_STREAM_PARALLEL 'p' +/* + * Copy data as text. This is the default. + */ +#define LOGICALREP_COPY_AS_TEXT 't' + +/* + * Copy data as binary if the binary options is enabled in the subscription. + */ +#define LOGICALREP_COPY_AS_BINARY 'b' + extern Subscription *GetSubscription(Oid subid, bool missing_ok); extern void FreeSubscription(Subscription *sub); extern void DisableSubscription(Oid subid); diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..7937defea5 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -27,5 +27,6 @@ extern ObjectAddress AlterSubscriptionOwner(const char *name, Oid newOwnerId); extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern char defGetCopyFormat(DefElem *def); #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 3f99b14394..045f34e710 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -114,18 +114,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -143,10 +143,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -163,10 +163,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/12345 | t (1 row) -- ok - with lsn = NONE @@ -175,10 +175,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | off | dbname=regress_doesnotexist2 | 0/0 | t (1 row) BEGIN; @@ -210,10 +210,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+--------------------+------------------------------+----------+------------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | local | dbname=regress_doesnotexist2 | 0/0 | t (1 row) -- rename back to keep the rest simple @@ -247,19 +247,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub; @@ -271,27 +271,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) -- fail - publication already exists @@ -306,10 +306,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) -- fail - publication used more than once @@ -324,10 +324,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) DROP SUBSCRIPTION regress_testsub; @@ -363,10 +363,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) --fail - alter of two_phase option not supported. @@ -375,10 +375,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -388,10 +388,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -404,22 +404,70 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | off | dbname=regress_doesnotexist | 0/0 | t (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); DROP SUBSCRIPTION regress_testsub; +-- fail - cannot set binary = false and copy_format = binary in a subscription +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (binary = false, copy_format = binary, connect = false, slot_name = NONE); +ERROR: binary = false and copy_format = binary are mutually exclusive options +-- now it works +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (binary = true, copy_format = binary, connect = false, slot_name = NONE); +WARNING: subscription was created, but is not connected +HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | b +(1 row) + +-- alter copy_format to text +ALTER SUBSCRIPTION regress_testsub SET (copy_format = text); +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t +(1 row) + +-- alter copy_format back to binary +ALTER SUBSCRIPTION regress_testsub SET (copy_format = binary); +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | b +(1 row) + +-- fail - cannot set binary = false if copy_format = binary +ALTER SUBSCRIPTION regress_testsub SET (binary = false); +ERROR: cannot set binary = false for a subscription with copy_format = binary +-- now it works +ALTER SUBSCRIPTION regress_testsub SET (binary = false, copy_format = text); +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN | Copy Format +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+----------+------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | off | dbname=regress_doesnotexist | 0/0 | t +(1 row) + +-- fail - cannot set copy_format = binary for a subscription with binary = false +ALTER SUBSCRIPTION regress_testsub SET (copy_format = binary); +ERROR: cannot set copy_format = binary for a subscription with binary = false +DROP SUBSCRIPTION regress_testsub; RESET SESSION AUTHORIZATION; DROP ROLE regress_subscription_user; DROP ROLE regress_subscription_user2; diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index 7281f5fee2..875f3af6bf 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -286,6 +286,32 @@ ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); DROP SUBSCRIPTION regress_testsub; +-- fail - cannot set binary = false and copy_format = binary in a subscription +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (binary = false, copy_format = binary, connect = false, slot_name = NONE); + +-- now it works +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (binary = true, copy_format = binary, connect = false, slot_name = NONE); +\dRs+ + +-- alter copy_format to text +ALTER SUBSCRIPTION regress_testsub SET (copy_format = text); +\dRs+ + +-- alter copy_format back to binary +ALTER SUBSCRIPTION regress_testsub SET (copy_format = binary); +\dRs+ + +-- fail - cannot set binary = false if copy_format = binary +ALTER SUBSCRIPTION regress_testsub SET (binary = false); + +-- now it works +ALTER SUBSCRIPTION regress_testsub SET (binary = false, copy_format = text); +\dRs+ + +-- fail - cannot set copy_format = binary for a subscription with binary = false +ALTER SUBSCRIPTION regress_testsub SET (copy_format = binary); +DROP SUBSCRIPTION regress_testsub; + RESET SESSION AUTHORIZATION; DROP ROLE regress_subscription_user; DROP ROLE regress_subscription_user2; diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index 6b5853b80b..02db58dc3b 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -19,6 +19,11 @@ my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->start; +# Create binary subscriber node +my $node_subscriber_binary = PostgreSQL::Test::Cluster->new('subscriber_binary'); +$node_subscriber_binary->init(allows_streaming => 'logical'); +$node_subscriber_binary->start; + # Create some preexisting content on publisher my $ddl = qq( CREATE EXTENSION hstore WITH SCHEMA public; @@ -104,6 +109,100 @@ my $ddl = qq( # Setup structure on both nodes $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +$node_subscriber_binary->safe_psql('postgres', $ddl); + +# Insert initial test data +$node_publisher->safe_psql( + 'postgres', qq( + -- test_tbl_one_array_col + INSERT INTO tst_one_array (a, b) VALUES + (1, '{1, 2, 3}'), + (2, '{2, 3, 1}'); + + -- test_tbl_arrays + INSERT INTO tst_arrays (a, b, c, d) VALUES + ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), + ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'); + + -- test_tbl_single_enum + INSERT INTO tst_one_enum (a, b) VALUES + (1, 'a'), + (2, 'b'); + + -- test_tbl_enums + INSERT INTO tst_enums (a, b) VALUES + ('a', '{b, c}'), + ('b', '{c, a}'); + + -- test_tbl_single_composites + INSERT INTO tst_one_comp (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composites + INSERT INTO tst_comps (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]); + + -- test_tbl_composite_with_enums + INSERT INTO tst_comp_enum (a, b) VALUES + (1, ROW(1.0, 'a', 1)), + (2, ROW(2.0, 'b', 2)); + + -- test_tbl_composite_with_enums_array + INSERT INTO tst_comp_enum_array (a, b) VALUES + (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), + (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]); + + -- test_tbl_composite_with_single_enums_array_in_composite + INSERT INTO tst_comp_one_enum_array (a, b) VALUES + (1, ROW(1.0, '{a, b, c}', 1)), + (2, ROW(2.0, '{a, b, c}', 2)); + + -- test_tbl_composite_with_enums_array_in_composite + INSERT INTO tst_comp_enum_what (a, b) VALUES + (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), + (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]); + + -- test_tbl_mixed_composites + INSERT INTO tst_comp_mix_array (a, b) VALUES + (ROW( + ROW(1,'a',1), + ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], + 'a', + '{a,b,NULL,c}'), + ARRAY[ + ROW( + ROW(1,'a',1), + ARRAY[ + ROW(1,'a',1)::tst_comp_basic_t, + ROW(2,'b',2)::tst_comp_basic_t, + NULL + ], + 'a', + '{a,b,c}' + )::tst_comp_mix_t + ] + ); + + -- test_tbl_range + INSERT INTO tst_range (a, b) VALUES + (1, '[1, 10]'), + (2, '[2, 20]'); + + -- test_tbl_range_array + INSERT INTO tst_range_array (a, b, c) VALUES + (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), + (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'); + + -- tst_hstore + INSERT INTO tst_hstore (a, b) VALUES + (1, '"a"=>"1"'), + (2, '"zzz"=>"foo"'); + + -- tst_dom_constr + INSERT INTO tst_dom_constr VALUES (10); +)); # Setup logical replication my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; @@ -113,89 +212,126 @@ $node_publisher->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_slot)" ); +$node_subscriber_binary->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub_binary CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (slot_name = tap_sub_binary_slot, binary = true, copy_format = 'binary')" +); # Wait for initial sync to finish as well $node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); +$node_subscriber_binary->wait_for_subscription_sync($node_publisher, 'tap_sub_binary'); -# Insert initial test data +my $sync_check = qq( + SET timezone = '+2'; + SELECT a, b FROM tst_one_array ORDER BY a; + SELECT a, b, c, d FROM tst_arrays ORDER BY a; + SELECT a, b FROM tst_one_enum ORDER BY a; + SELECT a, b FROM tst_enums ORDER BY a; + SELECT a, b FROM tst_one_comp ORDER BY a; + SELECT a, b FROM tst_comps ORDER BY a; + SELECT a, b FROM tst_comp_enum ORDER BY a; + SELECT a, b FROM tst_comp_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_one_enum_array ORDER BY a; + SELECT a, b FROM tst_comp_enum_what ORDER BY a; + SELECT a, b FROM tst_comp_mix_array ORDER BY a; + SELECT a, b FROM tst_range ORDER BY a; + SELECT a, b, c FROM tst_range_array ORDER BY a; + SELECT a, b FROM tst_hstore ORDER BY a; +); + +# Check the synced data on subscribers +my $result = $node_subscriber->safe_psql('postgres', $sync_check); +my $result_binary = $node_subscriber_binary->safe_psql('postgres', $sync_check); + +my $sync_result = '1|{1,2,3} +2|{2,3,1} +{1,2,3}|{a,b,c}|{1.1,2.2,3.3}|{"1 day","2 days","3 days"} +{2,3,1}|{b,c,a}|{2.2,3.3,1.1}|{00:02:00,00:03:00,00:01:00} +1|a +2|b +a|{b,c} +b|{c,a} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,a,1) +2|(2,b,2) +(1,a,1)|{"(1,a,1)"} +(2,b,2)|{"(2,b,2)"} +1|(1,"{a,b,c}",1) +2|(2,"{a,b,c}",2) +(1,"{a,b,c}",1)|{"(1,\"{a,b,c}\",1)"} +(2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} +("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +1|[1,11) +2|[2,21) +1|["2014-08-04 00:00:00+02",infinity)|{"[1,3)","[10,21)"} +2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} +1|"a"=>"1" +2|"zzz"=>"foo"'; + +is( $result, $sync_result, 'check initial sync on subscriber'); +is( $result_binary, $sync_result, 'check initial sync on subscriber in binary'); + +# Insert test data for update to apply changes $node_publisher->safe_psql( 'postgres', qq( -- test_tbl_one_array_col INSERT INTO tst_one_array (a, b) VALUES - (1, '{1, 2, 3}'), - (2, '{2, 3, 1}'), (3, '{3, 2, 1}'), (4, '{4, 3, 2}'), (5, '{5, NULL, 3}'); -- test_tbl_arrays INSERT INTO tst_arrays (a, b, c, d) VALUES - ('{1, 2, 3}', '{"a", "b", "c"}', '{1.1, 2.2, 3.3}', '{"1 day", "2 days", "3 days"}'), - ('{2, 3, 1}', '{"b", "c", "a"}', '{2.2, 3.3, 1.1}', '{"2 minutes", "3 minutes", "1 minute"}'), ('{3, 1, 2}', '{"c", "a", "b"}', '{3.3, 1.1, 2.2}', '{"3 years", "1 year", "2 years"}'), ('{4, 1, 2}', '{"d", "a", "b"}', '{4.4, 1.1, 2.2}', '{"4 years", "1 year", "2 years"}'), ('{5, NULL, NULL}', '{"e", NULL, "b"}', '{5.5, 1.1, NULL}', '{"5 years", NULL, NULL}'); -- test_tbl_single_enum INSERT INTO tst_one_enum (a, b) VALUES - (1, 'a'), - (2, 'b'), (3, 'c'), (4, 'd'), (5, NULL); -- test_tbl_enums INSERT INTO tst_enums (a, b) VALUES - ('a', '{b, c}'), - ('b', '{c, a}'), ('c', '{b, a}'), ('d', '{c, b}'), ('e', '{d, NULL}'); -- test_tbl_single_composites INSERT INTO tst_one_comp (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, NULL, 5)); -- test_tbl_composites INSERT INTO tst_comps (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_basic_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_basic_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_basic_t]), (ROW(4.0, 'd', 4), ARRAY[ROW(4, 'd', 3)::tst_comp_basic_t]), (ROW(5.0, 'e', NULL), ARRAY[NULL, ROW(5, NULL, 5)::tst_comp_basic_t]); -- test_tbl_composite_with_enums INSERT INTO tst_comp_enum (a, b) VALUES - (1, ROW(1.0, 'a', 1)), - (2, ROW(2.0, 'b', 2)), (3, ROW(3.0, 'c', 3)), (4, ROW(4.0, 'd', 4)), (5, ROW(NULL, 'e', NULL)); -- test_tbl_composite_with_enums_array INSERT INTO tst_comp_enum_array (a, b) VALUES - (ROW(1.0, 'a', 1), ARRAY[ROW(1, 'a', 1)::tst_comp_enum_t]), - (ROW(2.0, 'b', 2), ARRAY[ROW(2, 'b', 2)::tst_comp_enum_t]), (ROW(3.0, 'c', 3), ARRAY[ROW(3, 'c', 3)::tst_comp_enum_t]), (ROW(4.0, 'd', 3), ARRAY[ROW(3, 'd', 3)::tst_comp_enum_t]), (ROW(5.0, 'e', 3), ARRAY[ROW(3, 'e', 3)::tst_comp_enum_t, NULL]); -- test_tbl_composite_with_single_enums_array_in_composite INSERT INTO tst_comp_one_enum_array (a, b) VALUES - (1, ROW(1.0, '{a, b, c}', 1)), - (2, ROW(2.0, '{a, b, c}', 2)), (3, ROW(3.0, '{a, b, c}', 3)), (4, ROW(4.0, '{c, b, d}', 4)), (5, ROW(5.0, '{NULL, e, NULL}', 5)); -- test_tbl_composite_with_enums_array_in_composite INSERT INTO tst_comp_enum_what (a, b) VALUES - (ROW(1.0, '{a, b, c}', 1), ARRAY[ROW(1, '{a, b, c}', 1)::tst_comp_enum_array_t]), - (ROW(2.0, '{b, c, a}', 2), ARRAY[ROW(2, '{b, c, a}', 1)::tst_comp_enum_array_t]), (ROW(3.0, '{c, a, b}', 1), ARRAY[ROW(3, '{c, a, b}', 1)::tst_comp_enum_array_t]), (ROW(4.0, '{c, b, d}', 4), ARRAY[ROW(4, '{c, b, d}', 4)::tst_comp_enum_array_t]), (ROW(5.0, '{c, NULL, b}', NULL), ARRAY[ROW(5, '{c, e, b}', 1)::tst_comp_enum_array_t]); @@ -203,10 +339,10 @@ $node_publisher->safe_psql( -- test_tbl_mixed_composites INSERT INTO tst_comp_mix_array (a, b) VALUES (ROW( - ROW(1,'a',1), - ARRAY[ROW(1,'a',1)::tst_comp_basic_t, ROW(2,'b',2)::tst_comp_basic_t], - 'a', - '{a,b,NULL,c}'), + ROW(2,'b',2), + ARRAY[ROW(2,'b',2)::tst_comp_basic_t, ROW(3,'c',3)::tst_comp_basic_t], + 'b', + '{b,c,NULL,d}'), ARRAY[ ROW( ROW(1,'a',1), @@ -223,36 +359,29 @@ $node_publisher->safe_psql( -- test_tbl_range INSERT INTO tst_range (a, b) VALUES - (1, '[1, 10]'), - (2, '[2, 20]'), (3, '[3, 30]'), (4, '[4, 40]'), (5, '[5, 50]'); -- test_tbl_range_array INSERT INTO tst_range_array (a, b, c) VALUES - (1, tstzrange('Mon Aug 04 00:00:00 2014 CEST'::timestamptz, 'infinity'), '{"[1,2]", "[10,20]"}'), - (2, tstzrange('Sat Aug 02 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[2,3]", "[20,30]"}'), (3, tstzrange('Fri Aug 01 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[3,4]"}'), (4, tstzrange('Thu Jul 31 00:00:00 2014 CEST'::timestamptz, 'Mon Aug 04 00:00:00 2014 CEST'::timestamptz), '{"[4,5]", NULL, "[40,50]"}'), (5, NULL, NULL); -- tst_hstore INSERT INTO tst_hstore (a, b) VALUES - (1, '"a"=>"1"'), - (2, '"zzz"=>"foo"'), (3, '"123"=>"321"'), (4, '"yellow horse"=>"moaned"'); -- tst_dom_constr - INSERT INTO tst_dom_constr VALUES (10); + INSERT INTO tst_dom_constr VALUES (11); )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -my $result = $node_subscriber->safe_psql( - 'postgres', qq( +my $initial_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -268,9 +397,13 @@ my $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $initial_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $initial_check); -is( $result, '1|{1,2,3} +my $initial_result = '1|{1,2,3} 2|{2,3,1} 3|{3,2,1} 4|{4,3,2} @@ -321,6 +454,7 @@ e|{d,NULL} (4,"{c,b,d}",4)|{"(4,\"{c,b,d}\",4)"} (5,"{c,NULL,b}",)|{"(5,\"{c,e,b}\",1)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[1,11) 2|[2,21) 3|[3,31) @@ -334,8 +468,10 @@ e|{d,NULL} 1|"a"=>"1" 2|"zzz"=>"foo" 3|"123"=>"321" -4|"yellow horse"=>"moaned"', - 'check replicated inserts on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $initial_result, 'check replicated inserts on subscriber'); +is( $result_binary, $initial_result, 'check replicated inserts on subscriber in binary'); # Run batch of updates $node_publisher->safe_psql( @@ -370,10 +506,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $update_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -389,9 +524,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $update_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $update_check); -is( $result, '1|{4,5,6} +my $update_result = '1|{4,5,6} 2|{2,3,1} 3|{3,2,1} 4|{4,5,6,1} @@ -442,6 +581,7 @@ e|{e,d} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} ("(1,a,1)","{""(1,a,1)"",""(2,b,2)""}",a,"{a,b,NULL,c}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")",NULL} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 1|[100,1001) 2|[2,21) 3|[3,31) @@ -455,8 +595,10 @@ e|{e,d} 1|"updated"=>"value" 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated updates on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $update_result, 'check replicated updates on subscriber'); +is( $result_binary, $update_result, 'check replicated updates on subscriber in binary'); # Run batch of deletes $node_publisher->safe_psql( @@ -490,10 +632,9 @@ $node_publisher->safe_psql( )); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); -# Check the data on subscriber -$result = $node_subscriber->safe_psql( - 'postgres', qq( +my $delete_check = qq( SET timezone = '+2'; SELECT a, b FROM tst_one_array ORDER BY a; SELECT a, b, c, d FROM tst_arrays ORDER BY a; @@ -509,9 +650,13 @@ $result = $node_subscriber->safe_psql( SELECT a, b FROM tst_range ORDER BY a; SELECT a, b, c FROM tst_range_array ORDER BY a; SELECT a, b FROM tst_hstore ORDER BY a; -)); +); + +# Check the data on subscribers +$result = $node_subscriber->safe_psql('postgres', $delete_check); +$result_binary = $node_subscriber_binary->safe_psql('postgres', $delete_check); -is( $result, '3|{3,2,1} +my $delete_result = '3|{3,2,1} 4|{4,5,6,1} 5|{4,5,6,1} {3,1,2}|{c,a,b}|{3.3,1.1,2.2}|{"3 years","1 year","2 years"} @@ -540,26 +685,34 @@ e|{e,d} (2,"{b,c,a}",2)|{"(2,\"{b,c,a}\",1)"} (4,"{c,b,d}",4)|{"(5,\"{a,b,c}\",5)"} (5,"{c,NULL,b}",)|{"(5,\"{a,b,c}\",5)"} +("(2,b,2)","{""(2,b,2)"",""(3,c,3)""}",b,"{b,c,NULL,d}")|{"(\"(1,a,1)\",\"{\"\"(1,a,1)\"\",\"\"(2,b,2)\"\",NULL}\",a,\"{a,b,c}\")"} 2|["2014-08-02 00:00:00+02","2014-08-04 00:00:00+02")|{"[2,4)","[20,31)"} 3|["2014-08-01 00:00:00+02","2014-08-04 00:00:00+02")|{"[3,5)"} 2|"updated"=>"value" 3|"also"=>"updated" -4|"yellow horse"=>"moaned"', - 'check replicated deletes on subscriber'); +4|"yellow horse"=>"moaned"'; + +is( $result, $delete_result, 'check replicated deletes on subscriber'); +is( $result_binary, $delete_result, 'check replicated deletes on subscriber in binary'); # Test a domain with a constraint backed by a SQL-language function, # which needs an active snapshot in order to operate. $node_publisher->safe_psql('postgres', - "INSERT INTO tst_dom_constr VALUES (11)"); + "INSERT INTO tst_dom_constr VALUES (12)"); $node_publisher->wait_for_catchup('tap_sub'); +$node_publisher->wait_for_catchup('tap_sub_binary'); + +my $domain_check = qq(SELECT sum(a) FROM tst_dom_constr); + +$result = $node_subscriber->safe_psql('postgres', $domain_check); +$result_binary = $node_subscriber->safe_psql('postgres', $domain_check); -$result = - $node_subscriber->safe_psql('postgres', - "SELECT sum(a) FROM tst_dom_constr"); -is($result, '21', 'sql-function constraint on domain'); +is( $result, '33', 'sql-function constraint on domain'); +is( $result_binary, '33', 'sql-function constraint on domain'); $node_subscriber->stop('fast'); +$node_subscriber_binary->stop('fast'); $node_publisher->stop('fast'); done_testing(); diff --git a/src/test/subscription/t/014_binary.pl b/src/test/subscription/t/014_binary.pl index e53e23da3e..4fee421733 100644 --- a/src/test/subscription/t/014_binary.pl +++ b/src/test/subscription/t/014_binary.pl @@ -36,6 +36,16 @@ my $ddl = qq( $node_publisher->safe_psql('postgres', $ddl); $node_subscriber->safe_psql('postgres', $ddl); +# Insert some content and make sure it's synced to subscriber +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO public.test_arrays (a, b, c) VALUES + ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'); + + INSERT INTO public.test_numerical (a, b, c, d) VALUES + (1, 1.2, 1.3, 10); + )); + # Configure logical replication $node_publisher->safe_psql('postgres', "CREATE PUBLICATION tpub FOR ALL TABLES"); @@ -48,27 +58,35 @@ $node_subscriber->safe_psql('postgres', # Ensure nodes are in sync with each other $node_subscriber->wait_for_subscription_sync($node_publisher, 'tsub'); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); + +is( $result, '1|1.2|1.3|10 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three}', 'check syned data on subscriber'); + # Insert some content and make sure it's replicated across $node_publisher->safe_psql( 'postgres', qq( INSERT INTO public.test_arrays (a, b, c) VALUES - ('{1,2,3}', '{1.1, 1.2, 1.3}', '{"one", "two", "three"}'), ('{3,1,2}', '{1.3, 1.1, 1.2}', '{"three", "one", "two"}'); INSERT INTO public.test_numerical (a, b, c, d) VALUES - (1, 1.2, 1.3, 10), (2, 2.2, 2.3, 20), (3, 3.2, 3.3, 30); )); $node_publisher->wait_for_catchup('tsub'); -my $result = $node_subscriber->safe_psql('postgres', - "SELECT a, b, c, d FROM test_numerical ORDER BY a"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, b, c, d FROM test_numerical ORDER BY a; + SELECT a, b, c FROM test_arrays ORDER BY a;"); is( $result, '1|1.2|1.3|10 2|2.2|2.3|20 -3|3.2|3.3|30', 'check replicated data on subscriber'); +3|3.2|3.3|30 +{1,2,3}|{1.1,1.2,1.3}|{one,two,three} +{3,1,2}|{1.3,1.1,1.2}|{three,one,two}', 'check replicated data on subscriber'); # Test updates as well $node_publisher->safe_psql( @@ -130,6 +148,60 @@ is( $result, '{1,2,3}|{42,1.2,1.3}| {2,3,1}|{1.2,1.3,1.1}|{two,three,one} {3,1,2}|{42,1.1,1.2}|', 'check replicated data on subscriber'); +# Test whether subscriber fails in case of column type mismatch +$ddl = qq( + CREATE TABLE public.test_cross_type (a SMALLINT); + INSERT INTO public.test_cross_type VALUES (1); +); +$node_publisher->safe_psql('postgres', $ddl); + +$ddl = qq( + CREATE TABLE public.test_cross_type (a INTEGER); +); +$node_subscriber->safe_psql('postgres', $ddl); + +# Change the format of inital copy to binary +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tsub SET (copy_format = binary)'); +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tsub REFRESH PUBLICATION'); + +# Cannot copy initial data in binary format because of type mismatch +$node_subscriber->wait_for_log(qr/ERROR: ( [A-Z0-9]+:)? insufficient data left in message/); + +# Reset the format of initial copy back to text +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tsub SET (copy_format = text)'); + +# It should be able to sync the table now +$node_subscriber->wait_for_subscription_sync($node_publisher,'tsub'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT a FROM test_cross_type ORDER BY a"); + +is( $result, '1', 'check initial sync of data with type mismatch'); + +# Change the format of inital copy to binary again to test the apply phase +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tsub SET (copy_format = binary)'); + +# Insert a new row +$node_publisher->safe_psql( + 'postgres', qq( + INSERT INTO public.test_cross_type (a) VALUES + (2); + )); + +# Cannot apply due to type mismatch +$node_subscriber->wait_for_log(qr/ERROR: ( [A-Z0-9]+:)? insufficient data left in message/); + +# Set binary =false and copy_format = text +$node_subscriber->safe_psql('postgres', 'ALTER SUBSCRIPTION tsub SET (binary = false, copy_format = text)'); + +$node_publisher->wait_for_catchup('tsub'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT a FROM public.test_cross_type ORDER BY a"); + +is( $result, '1 +2', 'check replication of data with type mismatch'); + $node_subscriber->stop('fast'); $node_publisher->stop('fast'); -- 2.25.1 ^ permalink raw reply [nested|flat] 33+ messages in thread
* RE: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-02-01 03:05 ` Takamichi Osumi (Fujitsu) <[email protected]> 2023-02-16 09:33 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2 siblings, 1 reply; 33+ messages in thread From: Takamichi Osumi (Fujitsu) @ 2023-02-01 03:05 UTC (permalink / raw) To: 'Melih Mutlu' <[email protected]>; Bharath Rupireddy <[email protected]>; +Cc: [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers On Monday, January 30, 2023 7:50 PM Melih Mutlu <[email protected]> wrote: > Thanks for reviewing. ... > Please see [1] and you'll get the following error in your case: > "ERROR: incorrect binary data format in logical replication column 1" Hi, thanks for sharing v7. (1) general comment I wondered if the addition of the new option/parameter can introduce some confusion to the users. case 1. When binary = true and copy_format = text, the table sync is conducted by text. case 2. When binary = false and copy_format = binary, the table sync is conducted by binary. (Note that the case 2 can be accepted after addressing the 3rd comment of Bharath-san in [1]. I agree with the 3rd comment by itself.) The name of the new subscription parameter looks confusing. How about "init_sync_format" or something ? (2) The commit message doesn't get updated. The commit message needs to mention the new subscription option. (3) whitespace errors. $ git am v7-0001-Allow-logical-replication-to-copy-table-in-binary.patch Applying: Allow logical replication to copy table in binary .git/rebase-apply/patch:95: trailing whitespace. copied to the subscriber. Supported formats are .git/rebase-apply/patch:101: trailing whitespace. that data will not be copied if <literal>copy_data = false</literal>. warning: 2 lines add whitespace errors. (4) pg_dump.c if (fout->remoteVersion >= 160000) - appendPQExpBufferStr(query, " s.suborigin\n"); + appendPQExpBufferStr(query, " s.suborigin,\n"); else - appendPQExpBuffer(query, " '%s' AS suborigin\n", LOGICALREP_ORIGIN_ANY); + appendPQExpBuffer(query, " '%s' AS suborigin,\n", LOGICALREP_ORIGIN_ANY); + + if (fout->remoteVersion >= 160000) + appendPQExpBufferStr(query, " s.subcopyformat\n"); + else + appendPQExpBuffer(query, " '%c' AS subcopyformat\n", LOGICALREP_COPY_AS_TEXT); This new branch for v16 can be made together with the previous same condition. (5) describe.c + + /* Copy format is only supported in v16 and higher */ + if (pset.sversion >= 160000) + appendPQExpBuffer(&buf, + ", subcopyformat AS \"%s\"\n", + gettext_noop("Copy Format")); Similarly to (4), this creates a new branch for v16. Please see the above codes of this part. (6) + * Extract the copy format value from a DefElem. + */ +char +defGetCopyFormat(DefElem *def) Shouldn't this function be static and remove the change of subscriptioncmds.h ? (7) catalogs.sgml The subcopyformat should be mentioned here and the current description for subbinary should be removed. (8) create_subscription.sgml + <literal>text</literal>. + + <literal>binary</literal> format can be selected only if Unnecessary blank line. [1] - https://www.postgresql.org/message-id/CALj2ACW5Oa7_v25iZb326UXvtM_tjQfw0Tc3hPJ8zN4FZqc9cw%40mail.gma... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-02-01 03:05 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> @ 2023-02-16 09:33 ` Melih Mutlu <[email protected]> 0 siblings, 0 replies; 33+ messages in thread From: Melih Mutlu @ 2023-02-16 09:33 UTC (permalink / raw) To: Takamichi Osumi (Fujitsu) <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi, Thanks for reviewing. Please see the v8 here [1] Takamichi Osumi (Fujitsu) <[email protected]>, 1 Şub 2023 Çar, 06:05 tarihinde şunu yazdı: > (1) general comment > > I wondered if the addition of the new option/parameter can introduce some > confusion to the users. > > case 1. When binary = true and copy_format = text, the table sync is > conducted by text. > case 2. When binary = false and copy_format = binary, the table sync is > conducted by binary. > (Note that the case 2 can be accepted after addressing the 3rd comment of > Bharath-san in [1]. > I agree with the 3rd comment by itself.) > I replied to Bharath's comment [1], can you please check to see if that makes sense? > The name of the new subscription parameter looks confusing. > How about "init_sync_format" or something ? > The option to enable initial sync is named "copy_data", so I named the new parameter as "copy_format" to refer to that copy meant by "copy_data". Maybe "copy_data_format" would be better. I can change it if you think it's better. > (2) The commit message doesn't get updated. > Done > (3) whitespace errors. > Done > (4) pg_dump.c > Done > (5) describe.c > Done > (6) > > + * Extract the copy format value from a DefElem. > + */ > +char > +defGetCopyFormat(DefElem *def) > > Shouldn't this function be static and remove the change of > subscriptioncmds.h ? > I wanted to make "defGetCopyFormat" be consistent with "defGetStreamingMode" since they're basically doing the same work for different parameters. And that function isn't static, so I didn't make "defGetCopyFormat" static too. > (7) catalogs.sgml > Done (8) create_subscription.sgml > Done Also; The latest patch doesn't get updated for more than two weeks > after some review comments. If you don't have time, > I would like to help updating the patch for you and other reviewers. > > Kindly let me know your status. > Sorry for the delay. This patch is currently one of my priorities. Hopefully I will share quicker updates from now on. [1] https://www.postgresql.org/message-id/CAGPVpCQYi9AYQSS%3DRmGgVNjz5ZEnLB8mACwd9aioVhLmbgiAMA%40mail.g... Thanks, -- Melih Mutlu Microsoft ^ permalink raw reply [nested|flat] 33+ messages in thread
* RE: Allow logical replication to copy tables in binary format 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> @ 2023-02-16 05:16 ` Takamichi Osumi (Fujitsu) <[email protected]> 2 siblings, 0 replies; 33+ messages in thread From: Takamichi Osumi (Fujitsu) @ 2023-02-16 05:16 UTC (permalink / raw) To: 'Melih Mutlu' <[email protected]>; Bharath Rupireddy <[email protected]>; +Cc: [email protected] <[email protected]>; Euler Taveira <[email protected]>; Amit Kapila <[email protected]>; pgsql-hackers Hi, Melih On Monday, January 30, 2023 7:50 PM Melih Mutlu <[email protected]> wrote: > Thanks for reviewing. ... > Well, with this patch, it will begin to fail in the table copy phase... The latest patch doesn't get updated for more than two weeks after some review comments. If you don't have time, I would like to help updating the patch for you and other reviewers. Kindly let me know your status. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 33+ messages in thread
end of thread, other threads:[~2023-02-16 09:33 UTC | newest] Thread overview: 33+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]> 2023-01-11 10:44 Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-11 16:34 ` Re: Allow logical replication to copy tables in binary format vignesh C <[email protected]> 2023-01-12 03:07 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-01-12 08:23 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-18 07:16 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-01-30 10:49 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-01-30 12:34 ` Re: Allow logical replication to copy tables in binary format Bharath Rupireddy <[email protected]> 2023-02-16 09:18 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-02-01 03:05 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]> 2023-02-16 09:33 ` Re: Allow logical replication to copy tables in binary format Melih Mutlu <[email protected]> 2023-02-16 05:16 ` RE: Allow logical replication to copy tables in binary format Takamichi Osumi (Fujitsu) <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox