public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 08/20] union{} with a CompressionAlgorithm alg
31+ messages / 5 participants
[nested] [flat]
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/7] union with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 219 ++++++++++++++++------------------
1 file changed, 106 insertions(+), 113 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 10db22ff88..ae1ae7b0d0 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -614,22 +614,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
+ } u;
};
@@ -730,13 +735,7 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc(sizeof(cfp));
- fp->uncompressedfp = NULL;
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
-#endif
-#ifdef HAVE_LIBZSTD
- fp->zstd.fp = NULL;
-#endif
+ fp->alg = compression->alg;
switch (compression->alg)
{
@@ -749,15 +748,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -769,29 +768,29 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
+ fp->u.zstd.fp = fopen(path, mode);
// XXX: save the compression params
- if (fp->zstd.fp == NULL)
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
else if (strchr(mode, 'w'))
{
- fp->zstd.dstream = NULL;
- fp->zstd.output.size = ZSTD_CStreamOutSize(); // XXX
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZSTD_createCStream();
- if (fp->zstd.cstream == NULL)
+ fp->u.zstd.dstream = NULL;
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize(); // XXX
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZSTD_createCStream();
+ if (fp->u.zstd.cstream == NULL)
fatal("could not initialize compression library");
}
else if (strchr(mode, 'r'))
{
- fp->zstd.cstream = NULL;
- fp->zstd.input.size = ZSTD_DStreamOutSize(); // XXX
- fp->zstd.input.src = pg_malloc0(fp->zstd.input.size);
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.cstream = NULL;
+ fp->u.zstd.input.size = ZSTD_DStreamOutSize(); // XXX
+ fp->u.zstd.input.src = pg_malloc0(fp->u.zstd.input.size);
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
@@ -799,8 +798,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -824,26 +823,26 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
size_t res, cnt;
@@ -852,7 +851,7 @@ cfread(void *ptr, int size, cfp *fp)
output->pos = 0;
/* read compressed data */
- while ((cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp)))
+ while ((cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp)))
{
input->size = cnt;
input->pos = 0;
@@ -860,7 +859,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0 || output->pos == output->size)
break;
if (ZSTD_isError(res))
@@ -871,16 +870,13 @@ cfread(void *ptr, int size, cfp *fp)
break; /* We read all the data that fits */
}
- ret = output->pos;
+ return output->pos;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -888,16 +884,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -907,21 +903,20 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
return size;
}
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -930,39 +925,38 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
}
fprintf(stderr, "cfgetc %d\n", ret);
+ return ret;
}
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->u.fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -970,12 +964,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
int res;
res = cfread(buf, len, fp);
@@ -984,9 +978,9 @@ cfgets(cfp *fp, char *buf, int len)
*strchr(buf, '\n') = '\0';
return res > 0 ? buf : 0;
}
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+
+ return fgets(buf, len, fp->u.fp);
}
int
@@ -1000,56 +994,55 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
+ return result;
}
- else
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -1057,26 +1050,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
- else
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
--
2.17.0
--bgLLobvf7eP6VP5c
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0006-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/7] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 218 ++++++++++++++++------------------
1 file changed, 105 insertions(+), 113 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 421b8e04cb..06005c3fba 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -619,23 +619,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
#ifdef HAVE_LIBZ
@@ -726,13 +730,7 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc(sizeof(cfp));
- fp->uncompressedfp = NULL;
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
-#endif
-#ifdef HAVE_LIBZSTD
- fp->zstd.fp = NULL;
-#endif
+ fp->alg = compression->alg;
switch (compression->alg)
{
@@ -745,15 +743,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -765,26 +763,26 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
else if (strchr(mode, 'w'))
{
- fp->zstd.dstream = NULL;
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.dstream = NULL;
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.cstream = NULL;
- fp->zstd.input.size = ZSTD_DStreamInSize();
- fp->zstd.input.src = pg_malloc0(fp->zstd.input.size);
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.cstream = NULL;
+ fp->u.zstd.input.size = ZSTD_DStreamInSize();
+ fp->u.zstd.input.src = pg_malloc0(fp->u.zstd.input.size);
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
@@ -792,8 +790,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -817,26 +815,26 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
size_t res, cnt;
@@ -845,7 +843,7 @@ cfread(void *ptr, int size, cfp *fp)
output->pos = 0;
/* read compressed data */
- while ((cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp)))
+ while ((cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp)))
{
input->size = cnt;
input->pos = 0;
@@ -853,7 +851,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0 || output->pos == output->size)
break;
if (ZSTD_isError(res))
@@ -864,16 +862,13 @@ cfread(void *ptr, int size, cfp *fp)
break; /* We read all the data that fits */
}
- ret = output->pos;
+ return output->pos;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -881,16 +876,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -900,21 +895,20 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
return size;
}
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -923,39 +917,38 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
}
fprintf(stderr, "cfgetc %d\n", ret);
+ return ret;
}
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->u.fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -963,12 +956,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
int res;
res = cfread(buf, len, fp);
@@ -977,9 +970,9 @@ cfgets(cfp *fp, char *buf, int len)
*strchr(buf, '\n') = '\0';
return res > 0 ? buf : 0;
}
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+
+ return fgets(buf, len, fp->u.fp);
}
int
@@ -993,56 +986,55 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
+ return result;
}
- else
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -1050,26 +1042,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
- else
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
--
2.17.0
--w1A23YewkF9s+fLd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0006-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 08/20] union{} with a CompressionAlgorithm alg
@ 2020-12-21 06:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-21 06:11 UTC (permalink / raw)
---
src/bin/pg_dump/compress_io.c | 200 ++++++++++++++++++----------------
src/bin/pg_dump/pg_dump.c | 2 +-
2 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index fa94148cdf..e07436bc21 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -651,23 +651,27 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
+ CompressionAlgorithm alg;
+
+ union {
+ FILE *fp;
+
#ifdef HAVE_LIBZ
- gzFile compressedfp;
+ gzFile gzfp;
#endif
-#ifdef HAVE_LIBZSTD // XXX: this should be a union with a CompressionAlgorithm alg?
- /* This is a normal file to which we read/write compressed data */
- struct {
- FILE *fp;
- // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
- ZSTD_CStream *cstream;
- ZSTD_DStream *dstream;
- ZSTD_outBuffer output;
- ZSTD_inBuffer input;
- } zstd;
+#ifdef HAVE_LIBZSTD
+ struct {
+ /* This is a normal file to which we read/write compressed data */
+ FILE *fp;
+ // XXX: use one separate ZSTD_CStream per thread: disable on windows ?
+ ZSTD_CStream *cstream;
+ ZSTD_DStream *dstream;
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ } zstd;
#endif
-
+ } u;
};
static int hasSuffix(const char *filename);
@@ -754,6 +758,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -765,15 +771,15 @@ cfopen(const char *path, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzopen(path, mode_compression);
+ fp->u.gzfp = gzopen(path, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
+ fp->u.gzfp = gzopen(path, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -783,8 +789,8 @@ cfopen(const char *path, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fopen(path, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fopen(path, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -792,23 +798,23 @@ cfopen(const char *path, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fopen(path, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -830,6 +836,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
{
cfp *fp = pg_malloc0(sizeof(cfp));
+ fp->alg = compression->alg;
+
switch (compression->alg)
{
#ifdef HAVE_LIBZ
@@ -841,15 +849,15 @@ cfdopen(int fd, const char *mode, Compress *compression)
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
mode, compression->level);
- fp->compressedfp = gzdopen(fd, mode_compression);
+ fp->u.gzfp = gzdopen(fd, mode_compression);
}
else
{
/* don't specify a level, just use the zlib default */
- fp->compressedfp = gzdopen(fd, mode);
+ fp->u.gzfp = gzdopen(fd, mode);
}
- if (fp->compressedfp == NULL)
+ if (fp->u.gzfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -859,8 +867,8 @@ cfdopen(int fd, const char *mode, Compress *compression)
#ifdef HAVE_LIBZSTD
case COMPR_ALG_ZSTD:
- fp->zstd.fp = fdopen(fd, mode);
- if (fp->zstd.fp == NULL)
+ fp->u.zstd.fp = fdopen(fd, mode);
+ if (fp->u.zstd.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -868,23 +876,23 @@ cfdopen(int fd, const char *mode, Compress *compression)
else if (mode[0] == 'w' || mode[0] == 'a' ||
strchr(mode, '+') != NULL)
{
- fp->zstd.output.size = ZSTD_CStreamOutSize();
- fp->zstd.output.dst = pg_malloc0(fp->zstd.output.size);
- fp->zstd.cstream = ZstdCStreamParams(compression);
+ fp->u.zstd.output.size = ZSTD_CStreamOutSize();
+ fp->u.zstd.output.dst = pg_malloc0(fp->u.zstd.output.size);
+ fp->u.zstd.cstream = ZstdCStreamParams(compression);
}
else if (strchr(mode, 'r'))
{
- fp->zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
- fp->zstd.dstream = ZSTD_createDStream();
- if (fp->zstd.dstream == NULL)
+ fp->u.zstd.input.src = pg_malloc0(ZSTD_DStreamInSize());
+ fp->u.zstd.dstream = ZSTD_createDStream();
+ if (fp->u.zstd.dstream == NULL)
fatal("could not initialize compression library");
} // XXX else: bad mode
return fp;
#endif
case COMPR_ALG_NONE:
- fp->uncompressedfp = fdopen(fd, mode);
- if (fp->uncompressedfp == NULL)
+ fp->u.fp = fdopen(fd, mode);
+ if (fp->u.fp == NULL)
{
free_keep_errno(fp);
fp = NULL;
@@ -908,13 +916,13 @@ cfread(void *ptr, int size, cfp *fp)
return 0;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
+ ret = gzread(fp->u.gzfp, ptr, size);
+ if (ret != size && !gzeof(fp->u.gzfp))
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
@@ -924,10 +932,10 @@ cfread(void *ptr, int size, cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t input_size = ZSTD_DStreamInSize();
/* input_size is the allocated size */
size_t res, cnt;
@@ -953,7 +961,7 @@ cfread(void *ptr, int size, cfp *fp)
/* read compressed data if we must produce more input */
if (input->pos == input->size)
{
- cnt = fread(unconstify(void *, input->src), 1, input_size, fp->zstd.fp);
+ cnt = fread(unconstify(void *, input->src), 1, input_size, fp->u.zstd.fp);
input->size = cnt;
/* If we have no input to consume, we're done */
@@ -968,7 +976,7 @@ cfread(void *ptr, int size, cfp *fp)
for ( ; input->pos < input->size; )
{
/* decompress */
- res = ZSTD_decompressStream(fp->zstd.dstream, output, input);
+ res = ZSTD_decompressStream(fp->u.zstd.dstream, output, input);
if (res == 0)
break; /* End of frame */
if (output->pos == output->size)
@@ -985,9 +993,9 @@ cfread(void *ptr, int size, cfp *fp)
}
#endif
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ ret = fread(ptr, 1, size, fp->u.fp);
+ if (ret != size && !feof(fp->u.fp))
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -995,16 +1003,16 @@ int
cfwrite(const void *ptr, int size, cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzwrite(fp->u.gzfp, ptr, size);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
size_t res, cnt;
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
input->src = ptr;
input->size = size;
@@ -1014,11 +1022,11 @@ cfwrite(const void *ptr, int size, cfp *fp)
while (input->pos != input->size)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_continue);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_continue);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
}
@@ -1027,7 +1035,7 @@ cfwrite(const void *ptr, int size, cfp *fp)
}
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->u.fp);
}
int
@@ -1036,12 +1044,12 @@ cfgetc(cfp *fp)
int ret;
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- ret = gzgetc(fp->compressedfp);
+ ret = gzgetc(fp->u.gzfp);
if (ret == EOF)
{
- if (!gzeof(fp->compressedfp))
+ if (!gzeof(fp->u.gzfp))
fatal("could not read from input file: %s", strerror(errno));
else
fatal("could not read from input file: end of file");
@@ -1051,11 +1059,11 @@ cfgetc(cfp *fp)
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
if (cfread(&ret, 1, fp) != 1)
{
- if (feof(fp->zstd.fp))
+ if (feof(fp->u.zstd.fp))
fatal("could not read from input file: end of file");
else
fatal("could not read from input file: %s", strerror(errno));
@@ -1064,9 +1072,9 @@ cfgetc(cfp *fp)
}
#endif
- ret = fgetc(fp->uncompressedfp);
+ ret = fgetc(fp->u.fp);
if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ READ_ERROR_EXIT(fp->u.fp);
return ret;
}
@@ -1074,11 +1082,12 @@ char *
cfgets(cfp *fp, char *buf, int len)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzgets(fp->u.gzfp, buf, len);
#endif
+
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
/*
* Read one byte at a time until newline or EOF.
@@ -1102,7 +1111,7 @@ cfgets(cfp *fp, char *buf, int len)
}
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->u.fp);
}
/* Close the given compressed or uncompressed stream; return 0 on success. */
@@ -1117,54 +1126,54 @@ cfclose(cfp *fp)
return EOF;
}
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
+ result = gzclose(fp->u.gzfp);
+ fp->u.gzfp = NULL;
return result;
}
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
+ if (fp->alg == COMPR_ALG_ZSTD)
{
- ZSTD_outBuffer *output = &fp->zstd.output;
- ZSTD_inBuffer *input = &fp->zstd.input;
+ ZSTD_outBuffer *output = &fp->u.zstd.output;
+ ZSTD_inBuffer *input = &fp->u.zstd.input;
size_t res, cnt;
- if (fp->zstd.cstream)
+ if (fp->u.zstd.cstream)
{
for (;;)
{
output->pos = 0;
- res = ZSTD_compressStream2(fp->zstd.cstream, output, input, ZSTD_e_end);
+ res = ZSTD_compressStream2(fp->u.zstd.cstream, output, input, ZSTD_e_end);
if (ZSTD_isError(res))
fatal("could not compress data: %s", ZSTD_getErrorName(res));
- cnt = fwrite(output->dst, 1, output->pos, fp->zstd.fp);
+ cnt = fwrite(output->dst, 1, output->pos, fp->u.zstd.fp);
if (cnt != output->pos)
fatal("could not write data: %s", strerror(errno));
if (res == 0)
break;
}
- ZSTD_freeCStream(fp->zstd.cstream);
- pg_free(fp->zstd.output.dst);
+ ZSTD_freeCStream(fp->u.zstd.cstream);
+ pg_free(fp->u.zstd.output.dst);
}
- if (fp->zstd.dstream)
+ if (fp->u.zstd.dstream)
{
- ZSTD_freeDStream(fp->zstd.dstream);
- pg_free(unconstify(void *, fp->zstd.input.src));
+ ZSTD_freeDStream(fp->u.zstd.dstream);
+ pg_free(unconstify(void *, fp->u.zstd.input.src));
}
- result = fclose(fp->zstd.fp);
- fp->zstd.fp = NULL;
+ result = fclose(fp->u.zstd.fp);
+ fp->u.zstd.fp = NULL;
return result;
}
#endif
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ result = fclose(fp->u.fp);
+ fp->u.fp = NULL;
free_keep_errno(fp);
return result;
}
@@ -1173,25 +1182,26 @@ int
cfeof(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
+ if (fp->alg == COMPR_ALG_LIBZ)
+ return gzeof(fp->u.gzfp);
#endif
#ifdef HAVE_LIBZSTD
- if (fp->zstd.fp)
- return feof(fp->zstd.fp);
+ if (fp->alg == COMPR_ALG_ZSTD)
+ return feof(fp->u.zstd.fp);
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->u.fp);
}
const char *
get_cfp_error(cfp *fp)
{
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->alg == COMPR_ALG_LIBZ)
{
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->u.gzfp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 7c2f7a9ca3..5e009e5854 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -397,7 +397,7 @@ parse_compression(const char *optarg, Compress *compress)
const int default_compress_level[] = {
0, /* COMPR_ALG_NONE */
Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
- 0, // XXX: ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
+ 0, // #ifdef LIBZSTD ZSTD_CLEVEL_DEFAULT, /* COMPR_ALG_ZSTD */
};
compress->level = default_compress_level[compress->alg];
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0009-Move-zlib-into-the-union.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-01-30 22:05 Mark Wong <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Mark Wong @ 2026-01-30 22:05 UTC (permalink / raw)
To: Manni Wood <[email protected]>; +Cc: Nazir Bilal Yavuz <[email protected]>; KAZAR Ayoub <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
On Tue, Jan 13, 2026 at 06:20:27PM -0600, Manni Wood wrote:
> On Tue, Jan 13, 2026 at 1:12 PM Mark Wong <[email protected]> wrote:
>
> > On Fri, Jan 09, 2026 at 05:21:45PM +0300, Nazir Bilal Yavuz wrote:
> > > Were you able to understand why Mark's benchmark results are different
> > > from ours?
> >
> > Not yet... I had some guesses, which is why I suggested the processor
> > pinning
> > and using a ramdisk. But we haven't tried applying all of those to my
> > laptop,
> > which has 3 core types, or the POWER system, which may be interesting to
> > use a
> > ram disk on.
> >
> > I'm curious though, and admittedly haven't tried looking myself yet, about
> > how
> > the SIMD calls might look across different processor architectures. We'll
> > try
> > to get that on the POWER system soon...
> >
> > Regards,
> > Mark
>
> Hello!
>
> Nazir, I'm glad you are finding the benchmarks useful. I have more! :-)
>
> All of these benchmarks are all-in-RAM, because I do think that is the best
> way of getting closest to the theoretical best and worst case scenarios.
>
> My laptop:
>
> master: (852558b9)
>
> text, no special: 14996
> text, 1/3 special: 17270
> csv, no special: 18274
> csv, 1/3 special: 23852
>
> v3
>
> text, no special: 11282 (24.7% speedup)
> text, 1/3 special: 15748 (8.8% speedup) <-- I don't believe this but it's
> what I got
> csv, no special: 11571 (36.6% speedup)
> csv, 1/3 special: 19934 (16.4% speedup) <-- I don't believe this but it's
> what I got
>
> v4.2
>
> text, no special: 11139 (25.7% speedup)
> text, 1/3 special: 18900 (9.4% regression)
> csv, no special: 11490 (37.1% speedup)
> csv, 1/3 special: 26134 (9.5% regression)
>
> An AWS EC2 t2.2xlarge instance
>
> master: (852558b9)
>
> text, no special: 20677
> text, 1/3 special: 22660
> csv, no special: 24534
> csv, 1/3 special: 30999
>
> v3
>
> text, no special: 17534 (15.2% speedup)
> text, 1/3 special: 22816 (0.6% regression)
> csv, no special: 17664 (28.0% speedup)
> csv, 1/3 special: 29338 (5.3% speedup) <-- I don't believe this but it's
> what I got
>
> v4.2
>
> text, no special: 17459 (15.5% speedup)
> text, 1/3 special: 25051 (10.5% regression)
> csv, no special: 17574 (28.3% speedup)
> csv, 1/3 special: 32092 (3.5% regression)
>
> An AWS EC2 t4g.2xlarge instance (using ARM processor; first test of ARM
> processor!)
>
> master: (852558b9)
>
> text, no special: 22081
> text, 1/3 special: 25100
> csv, no special: 27296
> csv, 1/3 special: 32344
>
> v3
>
> text, no special: 17724 (19.7% speedup)
> text, 1/3 special: 27606 (9.9% regression) <-- yikes! We would want to test
> this more
> csv, no special: 17597 (35.5% speedup)
> csv, 1/3 special: 32597 (0.8% regression)
>
> v4.2
>
> text, no special: 17674 (20% speedup)
> text, 1/3 special: 25773 (2.6% regression) <-- this regression is less than
> for the v3 patch? Atypical...
> csv, no special: 17651 (35.3% speedup)
> csv, 1/3 special: 34055 (5.3% regression)
I'm still lagging behind a little I ran the v4.2 patches again applied to
71c11369 on the POWER system that I have access to, using Manni's copysimdperf
scripts to use a ramdisk and processor pinning.
text, no special: -2508 (30% speedup)
text, 1/3 special: -1753 (48% speedup)
csv, no special: 9264 (3% regression)
csv, 1/3 special: -4077 (0.3% speedup)
Using Manni's script makes me feel more confident about executing the tests in
the same way, so I don't know how concerning the difference in results are
compared to the other architectures.
> Yes, I think I agree with you that the everything-in-RAM benchmarks will
> make the regressions more pronounced, just like the everything-in-RAM
> benchmarks make the improvements more pronounced.
>
> I am not sure why the CSV regression, compared to the TXT regression (even
> for the v3 patch which has smaller regressions than the v4.2 patch) is
> usually worse. I probably should look over some flame graphs and see if I
> can find the place where the CSV-parsing code is so much slower. The CSV
> regression is actually a bit frustrating (at around 5%) because the TXT
> regression, at less than 1% (for the v3 patch) is so much easier to bare.
>
> Here are some copy-to benchmarks for the v4 patch that applies SIMD to the
> copy-to code.
>
> These were all-in-RAM tests.
>
> My laptop
>
> master: (852558b9)
>
> text, no special: 2948
> text, 1/3 special: 11258
> csv, no special: 6245
> csv, 1/3 special: 11258
>
> v4 (copy to)
>
> text, no special: 2126 (27.9% speedup)
> text, 1/3 special: 12080 (7.3% regression) <-- did not see such a big
> regression before
> csv, no special: 2432 (61.0% speedup)
> csv, 1/3 special: 12344 (4.0% regression) <-- did not see such a big
> regression before
>
> An AWS EC2 t2.2xlarge instance
>
> master: (852558b9)
>
> text, no special: 4647
> text, 1/3 special: 13865
> csv, no special: 5421
> csv, 1/3 special: 15284
>
> v4 (copy to)
>
> text, no special: 2460 (47.0% speedup)
> text, 1/3 special: 14023 (1.1% regression)
> csv, no special: 2667 (50.7% speedup)
> csv, 1/3 special: 15251 (0.2% speedup)
>
> An AWS EC2 t4g.2xlarge instance (using ARM processor; first test of ARM
> processor!)
>
> master: (852558b9)
>
> text, no special: 6951
> text, 1/3 special: 17857
> csv, no special: 7951
> csv, 1/3 special: 18504
>
> v4 (copy to)
>
> text, no special: 3372 (51.4% speedup)
> text, 1/3 special: 15713 (12.0% speedup)
> csv, no special: 3233 (59.3% speedup)
> csv, 1/3 special: 1622 (12.3% speedup)
>
> Once again, the v4 patch for copy-to seems like a clearer win, though, to
> be fair, there were regressions when running on my laptop. (I'm starting to
> think servers or desktops are better than laptops for testing these things,
> though maybe that's my bias: it just seems like the server results are
> always less surprising.)
>
> Hope you all continue to find these useful...
Regards,
Mark
--
Mark Wong <[email protected]>
EDB https://enterprisedb.com
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-02 20:45 KAZAR Ayoub <[email protected]>
parent: Mark Wong <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: KAZAR Ayoub @ 2026-02-02 20:45 UTC (permalink / raw)
To: Mark Wong <[email protected]>; Neil Conway <[email protected]>; +Cc: Manni Wood <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
Hello Mark,
On Fri, Jan 30, 2026 at 11:05 PM Mark Wong <[email protected]> wrote:
> On Tue, Jan 13, 2026 at 06:20:27PM -0600, Manni Wood wrote:
> > On Tue, Jan 13, 2026 at 1:12 PM Mark Wong <[email protected]> wrote:
> >
> > > On Fri, Jan 09, 2026 at 05:21:45PM +0300, Nazir Bilal Yavuz wrote:
> > > > Were you able to understand why Mark's benchmark results are
> different
> > > > from ours?
> > >
> > > Not yet... I had some guesses, which is why I suggested the processor
> > > pinning
> > > and using a ramdisk. But we haven't tried applying all of those to my
> > > laptop,
> > > which has 3 core types, or the POWER system, which may be interesting
> to
> > > use a
> > > ram disk on.
> > >
> > > I'm curious though, and admittedly haven't tried looking myself yet,
> about
> > > how
> > > the SIMD calls might look across different processor architectures.
> We'll
> > > try
> > > to get that on the POWER system soon...
> > >
> > > Regards,
> > > Mark
> >
> > Hello!
> >
> > Nazir, I'm glad you are finding the benchmarks useful. I have more! :-)
> >
> > All of these benchmarks are all-in-RAM, because I do think that is the
> best
> > way of getting closest to the theoretical best and worst case scenarios.
> >
> > My laptop:
> >
> > master: (852558b9)
> >
> > text, no special: 14996
> > text, 1/3 special: 17270
> > csv, no special: 18274
> > csv, 1/3 special: 23852
> >
> > v3
> >
> > text, no special: 11282 (24.7% speedup)
> > text, 1/3 special: 15748 (8.8% speedup) <-- I don't believe this but it's
> > what I got
> > csv, no special: 11571 (36.6% speedup)
> > csv, 1/3 special: 19934 (16.4% speedup) <-- I don't believe this but it's
> > what I got
> >
> > v4.2
> >
> > text, no special: 11139 (25.7% speedup)
> > text, 1/3 special: 18900 (9.4% regression)
> > csv, no special: 11490 (37.1% speedup)
> > csv, 1/3 special: 26134 (9.5% regression)
> >
> > An AWS EC2 t2.2xlarge instance
> >
> > master: (852558b9)
> >
> > text, no special: 20677
> > text, 1/3 special: 22660
> > csv, no special: 24534
> > csv, 1/3 special: 30999
> >
> > v3
> >
> > text, no special: 17534 (15.2% speedup)
> > text, 1/3 special: 22816 (0.6% regression)
> > csv, no special: 17664 (28.0% speedup)
> > csv, 1/3 special: 29338 (5.3% speedup) <-- I don't believe this but it's
> > what I got
> >
> > v4.2
> >
> > text, no special: 17459 (15.5% speedup)
> > text, 1/3 special: 25051 (10.5% regression)
> > csv, no special: 17574 (28.3% speedup)
> > csv, 1/3 special: 32092 (3.5% regression)
> >
> > An AWS EC2 t4g.2xlarge instance (using ARM processor; first test of ARM
> > processor!)
> >
> > master: (852558b9)
> >
> > text, no special: 22081
> > text, 1/3 special: 25100
> > csv, no special: 27296
> > csv, 1/3 special: 32344
> >
> > v3
> >
> > text, no special: 17724 (19.7% speedup)
> > text, 1/3 special: 27606 (9.9% regression) <-- yikes! We would want to
> test
> > this more
> > csv, no special: 17597 (35.5% speedup)
> > csv, 1/3 special: 32597 (0.8% regression)
> >
> > v4.2
> >
> > text, no special: 17674 (20% speedup)
> > text, 1/3 special: 25773 (2.6% regression) <-- this regression is less
> than
> > for the v3 patch? Atypical...
> > csv, no special: 17651 (35.3% speedup)
> > csv, 1/3 special: 34055 (5.3% regression)
>
> I'm still lagging behind a little I ran the v4.2 patches again applied to
> 71c11369 on the POWER system that I have access to, using Manni's
> copysimdperf
> scripts to use a ramdisk and processor pinning.
>
> text, no special: -2508 (30% speedup)
> text, 1/3 special: -1753 (48% speedup)
> csv, no special: 9264 (3% regression)
> csv, 1/3 special: -4077 (0.3% speedup)
>
> Thanks for the benchmark, I'm a bit suspicious about this because I find
it illogical or at least highly unexpected for a 1/3 specials workload to
perform better than no specials !
or csv no special regressing ! because it's expected to take the simd path
for the whole processing, so it's supposed to perform better than master
(at least ...).
I wonder what the results look like for COPY TO case on POWER. If you can
try, that case is at least even more theoretically predictable.
Regards,
Ayoub
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-03 10:59 Nazir Bilal Yavuz <[email protected]>
parent: KAZAR Ayoub <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Nazir Bilal Yavuz @ 2026-02-03 10:59 UTC (permalink / raw)
To: KAZAR Ayoub <[email protected]>; +Cc: Mark Wong <[email protected]>; Neil Conway <[email protected]>; Manni Wood <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
Hi,
On Mon, 2 Feb 2026 at 23:45, KAZAR Ayoub <[email protected]> wrote:
>
>>
> Thanks for the benchmark, I'm a bit suspicious about this because I find it illogical or at least highly unexpected for a 1/3 specials workload to perform better than no specials !
> or csv no special regressing ! because it's expected to take the simd path for the whole processing, so it's supposed to perform better than master (at least ...).
I agree with you. I think it is highly unexpected that 1/3 specials
cases perform better than no special cases.
> I wonder what the results look like for COPY TO case on POWER. If you can try, that case is at least even more theoretically predictable.
Yes, that would be helpful.
--
Regards,
Nazir Bilal Yavuz
Microsoft
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-03 11:02 Nazir Bilal Yavuz <[email protected]>
parent: Nazir Bilal Yavuz <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Nazir Bilal Yavuz @ 2026-02-03 11:02 UTC (permalink / raw)
To: KAZAR Ayoub <[email protected]>; +Cc: Mark Wong <[email protected]>; Neil Conway <[email protected]>; Manni Wood <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
Hi,
Thank you for all your help!
This thread has a couple of discussions and it can be hard to follow;
I think it would be better if we can separate these discussions. This
would make it easier to follow the topics. What I have in my mind is:
* There are four patches in this thread, three of them COPY FROM
related: v3, v4 and v5. One of them is COPY TO related. So, I guess it
would be better to move COPY TO discussion to another thread.
* Manni suggested to stop benchmarking v4.2 COPY FROM patch, I agree
with him. Then it would be easier to address reviews and do
benchmarking.
* Kazar applied Neil's review on top of v3 and benchmarked, v3 seems
to perform better but I think flushing line_buf needs more
benchmarking.
* I am planning to create a new version (v6) of the patch, which will
be basically v3 with one less branching (moving !in_quote check under
is_csv and else). So that it would be easy to review and follow the
thread for new reviewers.
What do you think about these? Do you think these make sense and / or
do you have any other suggestions?
--
Regards,
Nazir Bilal Yavuz
Microsoft
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-03 12:06 KAZAR Ayoub <[email protected]>
parent: Nazir Bilal Yavuz <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: KAZAR Ayoub @ 2026-02-03 12:06 UTC (permalink / raw)
To: Nazir Bilal Yavuz <[email protected]>; +Cc: Mark Wong <[email protected]>; Neil Conway <[email protected]>; Manni Wood <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
Hello,
On Tue, Feb 3, 2026, 12:02 PM Nazir Bilal Yavuz <[email protected]> wrote:
> Hi,
>
> * There are four patches in this thread, three of them COPY FROM
> related: v3, v4 and v5. One of them is COPY TO related. So, I guess it
> would be better to move COPY TO discussion to another thread.
>
If there's possible discussions concerning COPY TO, then yes because i
don't find many concerns about COPY TO.
> * I am planning to create a new version (v6) of the patch, which will
> be basically v3 with one less branching (moving !in_quote check under
> is_csv and else). So that it would be easy to review and follow the
> thread for new reviewers.
>
Oh cool, looking forward to this.
Regards,
Ayoub
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-04 05:38 Manni Wood <[email protected]>
parent: KAZAR Ayoub <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Manni Wood @ 2026-02-04 05:38 UTC (permalink / raw)
To: KAZAR Ayoub <[email protected]>; +Cc: Nazir Bilal Yavuz <[email protected]>; Mark Wong <[email protected]>; Neil Conway <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
On Tue, Feb 3, 2026 at 6:06 AM KAZAR Ayoub <[email protected]> wrote:
> Hello,
>
> On Tue, Feb 3, 2026, 12:02 PM Nazir Bilal Yavuz <[email protected]>
> wrote:
>
>> Hi,
>>
>> * There are four patches in this thread, three of them COPY FROM
>> related: v3, v4 and v5. One of them is COPY TO related. So, I guess it
>> would be better to move COPY TO discussion to another thread.
>>
> If there's possible discussions concerning COPY TO, then yes because i
> don't find many concerns about COPY TO.
>
>> * I am planning to create a new version (v6) of the patch, which will
>> be basically v3 with one less branching (moving !in_quote check under
>> is_csv and else). So that it would be easy to review and follow the
>> thread for new reviewers.
>>
> Oh cool, looking forward to this.
>
> Regards,
> Ayoub
>
Hello!
I have run some more tests.
x86 tower PC
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
origin/master, origin/HEAD)
Text, no special characters 30468.4045ms
CSV, no special characters 35351.084ms
Text, with 1/3 escapes 32722.48625ms
CSV, with 1/3 quotes 44044.87575ms
x86 tower PC
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
origin/master, origin/HEAD)
plus
0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch
Text, no special characters 23538.6895ms 22.7439379% improvement
CSV, no special characters 23475.94ms 33.59202224% improvement
Text, with 1/3 escapes 33530.8395ms -2.470329558% regression
CSV, with 1/3 quotes 45268.28275ms -2.777637533 regression
x86 tower PC
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
origin/master, origin/HEAD)
plus
v5.1-0001-Simple-heuristic-for-SIMD-COPY-FROM.patch.patch
Text, no special characters 22728.42475ms 25.40329852% improvement
CSV, no special characters 22777.7805ms 35.56695319% improvement
Text, with 1/3 escapes 34542.75625ms -5.562749683% regression
CSV, with 1/3 quotes 45793.0095 -3.96898327 regression
arm raspberry pi 5
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
postgres/master, postgres/HEAD)
Text, no special characters 9476.60875ms
CSV, no special characters 11132.6405ms
Text, with 1/3 escapes 10765.8125ms
CSV, with 1/3 quotes 14055.28925ms
arm raspberry pi 5
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
postgres/master, postgres/HEAD)
with
0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch
Text, no special characters 7380.328ms 22.12057926% improvement
CSV, no special characters 7349.53475ms 33.98210649% improvement
Text, with 1/3 escapes 10350.6385ms 3.856411209% improvement
CSV, with 1/3 quotes 12407.22725ms 11.72556445% improvement
arm raspberry pi 5
commit 78bf28e3bf504db0eea5e3bcb3c43e9908108480 (HEAD -> master,
postgres/master, postgres/HEAD)
with
v5.1-0001-Simple-heuristic-for-SIMD-COPY-FROM.patch.patch
Text, no special characters 7379.0375ms 22.134197% improvement
CSV, no special characters 7411.73225ms 33.42341154% improvement
Text, with 1/3 escapes 11288.465ms -4.854742733% regression
CSV, with 1/3 quotes 15281.3355ms -8.723023968% regression
The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems nice!
On My x86 PC, it had the usual performance improvment of earlier patches,
but the regression seemed more similar for both text and csv inputs.
Unfortunately, the regression is about 2.5%, but maybe that is an
acceptable worst-case for an improvement of 22% for text inputs and 33% for
CSV inputs?
The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch looks even
better on my Raspberry Pi's arm processor: not only do we see a 22%
improvement for text and an almost 34% improvement for CSV, even the
worst-case scenarios show an almost 4% improvement for text and an 11.7%
improvement for CSV.
By comparison,
the v5.1-0001-Simple-heuristic-for-SIMD-COPY-FROM.patch.patch's worst-case
performance is poorer on both architectures.
I'd be curious to know if anyone else can reproduces these
numbers. 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems
like a real winner.
Best,
-Manni
--
-- Manni Wood EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-04 14:28 KAZAR Ayoub <[email protected]>
parent: Manni Wood <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: KAZAR Ayoub @ 2026-02-04 14:28 UTC (permalink / raw)
To: Manni Wood <[email protected]>; +Cc: Nazir Bilal Yavuz <[email protected]>; Mark Wong <[email protected]>; Neil Conway <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
Hello,
On Wed, Feb 4, 2026, 6:38 AM Manni Wood <[email protected]> wrote:
> The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems nice!
> On My x86 PC, it had the usual performance improvment of earlier patches,
> but the regression seemed more similar for both text and csv inputs.
> Unfortunately, the regression is about 2.5%, but maybe that is an
> acceptable worst-case for an improvement of 22% for text inputs and 33% for
> CSV inputs?
>
> The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch looks even
> better on my Raspberry Pi's arm processor: not only do we see a 22%
> improvement for text and an almost 34% improvement for CSV, even the
> worst-case scenarios show an almost 4% improvement for text and an 11.7%
> improvement for CSV.
>
> By comparison,
> the v5.1-0001-Simple-heuristic-for-SIMD-COPY-FROM.patch.patch's worst-case
> performance is poorer on both architectures.
>
> I'd be curious to know if anyone else can reproduces these
> numbers. 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems
> like a real winner.
>
Thanks for the benchmark Manni, i suppose this is the same threshold as
patch has (4096 bytes), have you tried any bigger values for the threshold
?
Because i'm still expecting less l1d cache misses and execution times the
more we increase the threshold (relatively to l1d cache size per core).
As per my previous not-so-stable numbers 28KB wasn't too bad.
Regards,
Ayoub
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Speed up COPY FROM text/CSV parsing using SIMD
@ 2026-02-04 15:07 Manni Wood <[email protected]>
parent: KAZAR Ayoub <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Manni Wood @ 2026-02-04 15:07 UTC (permalink / raw)
To: KAZAR Ayoub <[email protected]>; +Cc: Nazir Bilal Yavuz <[email protected]>; Mark Wong <[email protected]>; Neil Conway <[email protected]>; Nathan Bossart <[email protected]>; Andrew Dunstan <[email protected]>; Shinya Kato <[email protected]>; pgsql-hackers
On Wed, Feb 4, 2026 at 8:29 AM KAZAR Ayoub <[email protected]> wrote:
> Hello,
>
> On Wed, Feb 4, 2026, 6:38 AM Manni Wood <[email protected]>
> wrote:
>
>> The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems
>> nice! On My x86 PC, it had the usual performance improvment of earlier
>> patches, but the regression seemed more similar for both text and csv
>> inputs. Unfortunately, the regression is about 2.5%, but maybe that is an
>> acceptable worst-case for an improvement of 22% for text inputs and 33% for
>> CSV inputs?
>>
>> The 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch looks even
>> better on my Raspberry Pi's arm processor: not only do we see a 22%
>> improvement for text and an almost 34% improvement for CSV, even the
>> worst-case scenarios show an almost 4% improvement for text and an 11.7%
>> improvement for CSV.
>>
>> By comparison,
>> the v5.1-0001-Simple-heuristic-for-SIMD-COPY-FROM.patch.patch's worst-case
>> performance is poorer on both architectures.
>>
>> I'd be curious to know if anyone else can reproduces these
>> numbers. 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch seems
>> like a real winner.
>>
> Thanks for the benchmark Manni, i suppose this is the same threshold as
> patch has (4096 bytes), have you tried any bigger values for the threshold
> ?
> Because i'm still expecting less l1d cache misses and execution times the
> more we increase the threshold (relatively to l1d cache size per core).
> As per my previous not-so-stable numbers 28KB wasn't too bad.
>
>
> Regards,
> Ayoub
>
Ah, thanks for the prod, Ayoub. You are correct: The results in my previous
e-mail for the 0001-COPY-from-SIMD-v3-with-line_buf-periodic-refill.patch
patch are with LINE_BUF_FLUSH_AFTER set to its default of 4096. I will try
to measure what happens for larger LINE_BUF_FLUSH_AFTER values, hopefully
some time this week.
Best,
-Manni
--
-- Manni Wood EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 31+ messages in thread
end of thread, other threads:[~2026-02-04 15:07 UTC | newest]
Thread overview: 31+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 5/7] union with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 5/7] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2020-12-21 06:11 [PATCH 08/20] union{} with a CompressionAlgorithm alg Justin Pryzby <[email protected]>
2026-01-30 22:05 Re: Speed up COPY FROM text/CSV parsing using SIMD Mark Wong <[email protected]>
2026-02-02 20:45 ` Re: Speed up COPY FROM text/CSV parsing using SIMD KAZAR Ayoub <[email protected]>
2026-02-03 10:59 ` Re: Speed up COPY FROM text/CSV parsing using SIMD Nazir Bilal Yavuz <[email protected]>
2026-02-03 11:02 ` Re: Speed up COPY FROM text/CSV parsing using SIMD Nazir Bilal Yavuz <[email protected]>
2026-02-03 12:06 ` Re: Speed up COPY FROM text/CSV parsing using SIMD KAZAR Ayoub <[email protected]>
2026-02-04 05:38 ` Re: Speed up COPY FROM text/CSV parsing using SIMD Manni Wood <[email protected]>
2026-02-04 14:28 ` Re: Speed up COPY FROM text/CSV parsing using SIMD KAZAR Ayoub <[email protected]>
2026-02-04 15:07 ` Re: Speed up COPY FROM text/CSV parsing using SIMD Manni Wood <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox