agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 3/6] Optimize allocations in bringetbitmap 5+ messages / 2 participants [nested] [flat]
* [PATCH 3/6] Optimize allocations in bringetbitmap @ 2020-09-13 10:12 Tomas Vondra <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Tomas Vondra @ 2020-09-13 10:12 UTC (permalink / raw) The bringetbitmap function allocates memory for various purposes, which may be quite expensive, depending on the number of scan keys. Instead of allocating them separately, allocate one bit chunk of memory an carve it into smaller pieces as needed - all the pieces have the same lifespan, and it saves quite a bit of CPU and memory overhead. Author: Tomas Vondra <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/backend/access/brin/brin.c | 62 +++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index 14da9ed17f..3735c41788 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -373,6 +373,9 @@ bringetbitmap(IndexScanDesc scan, TIDBitmap *tbm) int *nkeys, *nnullkeys; int keyno; + char *ptr; + Size len; + char *tmp PG_USED_FOR_ASSERTS_ONLY; opaque = (BrinOpaque *) scan->opaque; bdesc = opaque->bo_bdesc; @@ -398,11 +401,50 @@ bringetbitmap(IndexScanDesc scan, TIDBitmap *tbm) * Make room for per-attribute lists of scan keys that we'll pass to the * consistent support procedure. We keep null and regular keys separate, * so that we can easily pass regular keys to the consistent function. + * + * To reduce the allocation overhead, we allocate one big chunk and then + * carve it into smaller arrays ourselves. All the pieces have exactly + * the same lifetime, so that's OK. */ - keys = palloc0(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts); - nullkeys = palloc0(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts); - nkeys = palloc0(sizeof(int) * bdesc->bd_tupdesc->natts); - nnullkeys = palloc0(sizeof(int) * bdesc->bd_tupdesc->natts); + len = + /* regular keys */ + MAXALIGN(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts) + + MAXALIGN(sizeof(ScanKey) * scan->numberOfKeys * bdesc->bd_tupdesc->natts) + + MAXALIGN(sizeof(int) * bdesc->bd_tupdesc->natts) + + /* NULL keys */ + MAXALIGN(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts) + + MAXALIGN(sizeof(ScanKey) * scan->numberOfKeys * bdesc->bd_tupdesc->natts) + + MAXALIGN(sizeof(int) * bdesc->bd_tupdesc->natts); + + ptr = palloc(len); + tmp = ptr; + + keys = (ScanKey **) ptr; + ptr += MAXALIGN(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts); + + nullkeys = (ScanKey **) ptr; + ptr += MAXALIGN(sizeof(ScanKey *) * bdesc->bd_tupdesc->natts); + + nkeys = (int *) ptr; + ptr += MAXALIGN(sizeof(int) * bdesc->bd_tupdesc->natts); + + nnullkeys = (int *) ptr; + ptr += MAXALIGN(sizeof(int) * bdesc->bd_tupdesc->natts); + + for (int i = 0; i < bdesc->bd_tupdesc->natts; i++) + { + keys[i] = (ScanKey *) ptr; + ptr += MAXALIGN(sizeof(ScanKey) * scan->numberOfKeys); + + nullkeys[i] = (ScanKey *) ptr; + ptr += MAXALIGN(sizeof(ScanKey) * scan->numberOfKeys); + } + + Assert(tmp + len == ptr); + + /* zero the number of keys */ + memset(nkeys, 0, sizeof(int) * bdesc->bd_tupdesc->natts); + memset(nnullkeys, 0, sizeof(int) * bdesc->bd_tupdesc->natts); /* * Preprocess the scan keys - split them into per-attribute arrays. @@ -438,9 +480,9 @@ bringetbitmap(IndexScanDesc scan, TIDBitmap *tbm) { FmgrInfo *tmp; - /* No key/null arrays for this attribute. */ - Assert((keys[keyattno - 1] == NULL) && (nkeys[keyattno - 1] == 0)); - Assert((nullkeys[keyattno - 1] == NULL) && (nnullkeys[keyattno - 1] == 0)); + /* First time we see this attribute, so no key/null keys. */ + Assert(nkeys[keyattno - 1] == 0); + Assert(nnullkeys[keyattno - 1] == 0); tmp = index_getprocinfo(idxRel, keyattno, BRIN_PROCNUM_CONSISTENT); @@ -451,17 +493,11 @@ bringetbitmap(IndexScanDesc scan, TIDBitmap *tbm) /* Add key to the proper per-attribute array. */ if (key->sk_flags & SK_ISNULL) { - if (!nullkeys[keyattno - 1]) - nullkeys[keyattno - 1] = palloc0(sizeof(ScanKey) * scan->numberOfKeys); - nullkeys[keyattno - 1][nnullkeys[keyattno - 1]] = key; nnullkeys[keyattno - 1]++; } else { - if (!keys[keyattno - 1]) - keys[keyattno - 1] = palloc0(sizeof(ScanKey) * scan->numberOfKeys); - keys[keyattno - 1][nkeys[keyattno - 1]] = key; nkeys[keyattno - 1]++; } -- 2.26.2 --------------310A2AE1CC4C2E2E77559E3D Content-Type: text/x-patch; charset=UTF-8; name="0004-BRIN-bloom-indexes-20210114-2.patch" Content-Transfer-Encoding: 8bit Content-Disposition: attachment; filename="0004-BRIN-bloom-indexes-20210114-2.patch" ^ permalink raw reply [nested|flat] 5+ messages in thread
* [PATCH] f!fixes for LZ4 @ 2023-02-24 20:07 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Justin Pryzby @ 2023-02-24 20:07 UTC (permalink / raw) --- doc/src/sgml/ref/pg_dump.sgml | 4 ++-- src/bin/pg_dump/compress_gzip.c | 7 ------- src/bin/pg_dump/compress_io.c | 2 +- src/bin/pg_dump/compress_io.h | 4 ++-- src/bin/pg_dump/compress_lz4.c | 4 ++-- src/bin/pg_dump/pg_backup_archiver.c | 4 ++-- src/bin/pg_dump/pg_dump.c | 2 -- src/bin/pg_dump/t/002_pg_dump.pl | 6 +++--- 8 files changed, 12 insertions(+), 21 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 49d218905fb..6fbe49f7ede 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -331,7 +331,7 @@ PostgreSQL documentation can read. A directory format archive can be manipulated with standard Unix tools; for example, files in an uncompressed archive can be compressed with the <application>gzip</application> or - <application>lz4</application>tool. + <application>lz4</application> tools. This format is compressed by default using <literal>gzip</literal> and also supports parallel dumps. </para> @@ -655,7 +655,7 @@ PostgreSQL documentation <para> Specify the compression method and/or the compression level to use. The compression method can be set to <literal>gzip</literal> or - <literal>lz4</literal> or <literal>none</literal> for no compression. + <literal>lz4</literal>, or <literal>none</literal> for no compression. A compression detail string can optionally be specified. If the detail string is an integer, it specifies the compression level. Otherwise, it should be a comma-separated list of items, each of the diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c index 0af65afeb4e..af68fd27a12 100644 --- a/src/bin/pg_dump/compress_gzip.c +++ b/src/bin/pg_dump/compress_gzip.c @@ -123,13 +123,6 @@ WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs, gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1); gzipcs->outsize = ZLIB_OUT_SIZE; - /* - * A level of zero simply copies the input one block at the time. This - * is probably not what the user wanted when calling this interface. - */ - if (cs->compression_spec.level == 0) - pg_fatal("requested to compress the archive yet no level was specified"); - if (deflateInit(zp, cs->compression_spec.level) != Z_OK) pg_fatal("could not initialize compression library: %s", zp->msg); diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index ce06f1eac9c..9239dbb2755 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -83,7 +83,7 @@ * used by the caller in an error message. */ char * -supports_compression(const pg_compress_specification compression_spec) +pgdump_supports_compression(const pg_compress_specification compression_spec) { const pg_compress_algorithm algorithm = compression_spec.algorithm; bool supported = false; diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index bbde2693915..46815fa2ebe 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,7 +21,7 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -extern char *supports_compression(const pg_compress_specification compression_spec); +extern char *pgdump_supports_compression(const pg_compress_specification compression_spec); /* * Prototype for callback function used in writeData() @@ -172,7 +172,7 @@ struct CompressFileHandle extern CompressFileHandle *InitCompressFileHandle(const pg_compress_specification compression_spec); /* - * Initialize a compress file stream. Deffer the compression algorithm + * Initialize a compress file stream. Infer the compression algorithm * from 'path', either by examining its suffix or by appending the supported * suffixes in 'path'. */ diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index fe1014e6e77..63e794cdc68 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -161,8 +161,8 @@ typedef struct LZ4File } LZ4File; /* - * LZ4 equivalent to feof() or gzeof(). The end of file is reached if there - * is no decompressed output in the overflow buffer and the end of the file + * LZ4 equivalent to feof() or gzeof(). Return true iff there is no + * decompressed output in the overflow buffer and the end of the backing file * is reached. */ static int diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 61ebb8fe85d..2063d6f239d 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -388,7 +388,7 @@ RestoreArchive(Archive *AHX) { if (te->hadDumper && (te->reqs & REQ_DATA) != 0) { - char *errmsg = supports_compression(AH->compression_spec); + char *errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) pg_fatal("cannot restore from compressed archive (%s)", errmsg); @@ -3745,7 +3745,7 @@ ReadHead(ArchiveHandle *AH) else AH->compression_spec.algorithm = PG_COMPRESSION_GZIP; - errmsg = supports_compression(AH->compression_spec); + errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) { pg_log_warning("archive is compressed, but this installation does not support compression (%s) -- no data will be available", diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 24ba936332d..ce2242195f3 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -733,8 +733,6 @@ main(int argc, char **argv) #ifdef HAVE_LIBZ parse_compress_specification(PG_COMPRESSION_GZIP, NULL, &compression_spec); -#else - /* Nothing to do in the default case */ #endif } diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 72b19ee6cde..ad7bc5c194b 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4248,10 +4248,10 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; my $run_db = 'postgres'; - # Skip command-level tests for gzip if there is no support for it. + # Skip command-level tests for gzip/lz4 if they're not supported. if ($pgdump_runs{$run}->{compile_option} && - ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || - ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4)) + (($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || + ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))) { note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support"; next; -- 2.34.1 --lqtP4FxvQo6hb6bg-- ^ permalink raw reply [nested|flat] 5+ messages in thread
* [PATCH] f!fixes for LZ4 @ 2023-02-24 20:07 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Justin Pryzby @ 2023-02-24 20:07 UTC (permalink / raw) --- doc/src/sgml/ref/pg_dump.sgml | 4 ++-- src/bin/pg_dump/compress_io.c | 2 +- src/bin/pg_dump/compress_io.h | 4 ++-- src/bin/pg_dump/compress_lz4.c | 4 ++-- src/bin/pg_dump/pg_backup_archiver.c | 4 ++-- src/bin/pg_dump/pg_dump.c | 2 -- src/bin/pg_dump/t/002_pg_dump.pl | 6 +++--- 7 files changed, 12 insertions(+), 14 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 49d218905fb..6fbe49f7ede 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -331,7 +331,7 @@ PostgreSQL documentation can read. A directory format archive can be manipulated with standard Unix tools; for example, files in an uncompressed archive can be compressed with the <application>gzip</application> or - <application>lz4</application>tool. + <application>lz4</application> tools. This format is compressed by default using <literal>gzip</literal> and also supports parallel dumps. </para> @@ -655,7 +655,7 @@ PostgreSQL documentation <para> Specify the compression method and/or the compression level to use. The compression method can be set to <literal>gzip</literal> or - <literal>lz4</literal> or <literal>none</literal> for no compression. + <literal>lz4</literal>, or <literal>none</literal> for no compression. A compression detail string can optionally be specified. If the detail string is an integer, it specifies the compression level. Otherwise, it should be a comma-separated list of items, each of the diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index ce06f1eac9c..9239dbb2755 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -83,7 +83,7 @@ * used by the caller in an error message. */ char * -supports_compression(const pg_compress_specification compression_spec) +pgdump_supports_compression(const pg_compress_specification compression_spec) { const pg_compress_algorithm algorithm = compression_spec.algorithm; bool supported = false; diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index bbde2693915..46815fa2ebe 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,7 +21,7 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -extern char *supports_compression(const pg_compress_specification compression_spec); +extern char *pgdump_supports_compression(const pg_compress_specification compression_spec); /* * Prototype for callback function used in writeData() @@ -172,7 +172,7 @@ struct CompressFileHandle extern CompressFileHandle *InitCompressFileHandle(const pg_compress_specification compression_spec); /* - * Initialize a compress file stream. Deffer the compression algorithm + * Initialize a compress file stream. Infer the compression algorithm * from 'path', either by examining its suffix or by appending the supported * suffixes in 'path'. */ diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index fe1014e6e77..63e794cdc68 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -161,8 +161,8 @@ typedef struct LZ4File } LZ4File; /* - * LZ4 equivalent to feof() or gzeof(). The end of file is reached if there - * is no decompressed output in the overflow buffer and the end of the file + * LZ4 equivalent to feof() or gzeof(). Return true iff there is no + * decompressed output in the overflow buffer and the end of the backing file * is reached. */ static int diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 61ebb8fe85d..2063d6f239d 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -388,7 +388,7 @@ RestoreArchive(Archive *AHX) { if (te->hadDumper && (te->reqs & REQ_DATA) != 0) { - char *errmsg = supports_compression(AH->compression_spec); + char *errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) pg_fatal("cannot restore from compressed archive (%s)", errmsg); @@ -3745,7 +3745,7 @@ ReadHead(ArchiveHandle *AH) else AH->compression_spec.algorithm = PG_COMPRESSION_GZIP; - errmsg = supports_compression(AH->compression_spec); + errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) { pg_log_warning("archive is compressed, but this installation does not support compression (%s) -- no data will be available", diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 24ba936332d..ce2242195f3 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -733,8 +733,6 @@ main(int argc, char **argv) #ifdef HAVE_LIBZ parse_compress_specification(PG_COMPRESSION_GZIP, NULL, &compression_spec); -#else - /* Nothing to do in the default case */ #endif } diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 72b19ee6cde..ad7bc5c194b 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4248,10 +4248,10 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; my $run_db = 'postgres'; - # Skip command-level tests for gzip if there is no support for it. + # Skip command-level tests for gzip/lz4 if they're not supported. if ($pgdump_runs{$run}->{compile_option} && - ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || - ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4)) + (($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || + ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))) { note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support"; next; -- 2.34.1 --p8BcnzLwh3ipgLRM-- ^ permalink raw reply [nested|flat] 5+ messages in thread
* [PATCH 1/4] f!fixes for LZ4 @ 2023-02-24 20:07 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Justin Pryzby @ 2023-02-24 20:07 UTC (permalink / raw) --- doc/src/sgml/ref/pg_dump.sgml | 4 ++-- src/bin/pg_dump/compress_gzip.c | 8 -------- src/bin/pg_dump/compress_io.c | 2 +- src/bin/pg_dump/compress_io.h | 4 ++-- src/bin/pg_dump/compress_lz4.c | 4 ++-- src/bin/pg_dump/pg_backup_archiver.c | 4 ++-- src/bin/pg_dump/pg_dump.c | 2 -- src/bin/pg_dump/t/002_pg_dump.pl | 6 +++--- 8 files changed, 12 insertions(+), 22 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 49d218905fb..6fbe49f7ede 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -331,7 +331,7 @@ PostgreSQL documentation can read. A directory format archive can be manipulated with standard Unix tools; for example, files in an uncompressed archive can be compressed with the <application>gzip</application> or - <application>lz4</application>tool. + <application>lz4</application> tools. This format is compressed by default using <literal>gzip</literal> and also supports parallel dumps. </para> @@ -655,7 +655,7 @@ PostgreSQL documentation <para> Specify the compression method and/or the compression level to use. The compression method can be set to <literal>gzip</literal> or - <literal>lz4</literal> or <literal>none</literal> for no compression. + <literal>lz4</literal>, or <literal>none</literal> for no compression. A compression detail string can optionally be specified. If the detail string is an integer, it specifies the compression level. Otherwise, it should be a comma-separated list of items, each of the diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c index 0af65afeb4e..52f41c2e58c 100644 --- a/src/bin/pg_dump/compress_gzip.c +++ b/src/bin/pg_dump/compress_gzip.c @@ -123,17 +123,9 @@ WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs, gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1); gzipcs->outsize = ZLIB_OUT_SIZE; - /* - * A level of zero simply copies the input one block at the time. This - * is probably not what the user wanted when calling this interface. - */ - if (cs->compression_spec.level == 0) - pg_fatal("requested to compress the archive yet no level was specified"); - if (deflateInit(zp, cs->compression_spec.level) != Z_OK) pg_fatal("could not initialize compression library: %s", zp->msg); - /* Just be paranoid - maybe End is called after Start, with no Write */ zp->next_out = gzipcs->outbuf; zp->avail_out = gzipcs->outsize; } diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index ce06f1eac9c..9239dbb2755 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -83,7 +83,7 @@ * used by the caller in an error message. */ char * -supports_compression(const pg_compress_specification compression_spec) +pgdump_supports_compression(const pg_compress_specification compression_spec) { const pg_compress_algorithm algorithm = compression_spec.algorithm; bool supported = false; diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index bbde2693915..46815fa2ebe 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,7 +21,7 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -extern char *supports_compression(const pg_compress_specification compression_spec); +extern char *pgdump_supports_compression(const pg_compress_specification compression_spec); /* * Prototype for callback function used in writeData() @@ -172,7 +172,7 @@ struct CompressFileHandle extern CompressFileHandle *InitCompressFileHandle(const pg_compress_specification compression_spec); /* - * Initialize a compress file stream. Deffer the compression algorithm + * Initialize a compress file stream. Infer the compression algorithm * from 'path', either by examining its suffix or by appending the supported * suffixes in 'path'. */ diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index fe1014e6e77..63e794cdc68 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -161,8 +161,8 @@ typedef struct LZ4File } LZ4File; /* - * LZ4 equivalent to feof() or gzeof(). The end of file is reached if there - * is no decompressed output in the overflow buffer and the end of the file + * LZ4 equivalent to feof() or gzeof(). Return true iff there is no + * decompressed output in the overflow buffer and the end of the backing file * is reached. */ static int diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 61ebb8fe85d..2063d6f239d 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -388,7 +388,7 @@ RestoreArchive(Archive *AHX) { if (te->hadDumper && (te->reqs & REQ_DATA) != 0) { - char *errmsg = supports_compression(AH->compression_spec); + char *errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) pg_fatal("cannot restore from compressed archive (%s)", errmsg); @@ -3745,7 +3745,7 @@ ReadHead(ArchiveHandle *AH) else AH->compression_spec.algorithm = PG_COMPRESSION_GZIP; - errmsg = supports_compression(AH->compression_spec); + errmsg = pgdump_supports_compression(AH->compression_spec); if (errmsg) { pg_log_warning("archive is compressed, but this installation does not support compression (%s) -- no data will be available", diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 24ba936332d..ce2242195f3 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -733,8 +733,6 @@ main(int argc, char **argv) #ifdef HAVE_LIBZ parse_compress_specification(PG_COMPRESSION_GZIP, NULL, &compression_spec); -#else - /* Nothing to do in the default case */ #endif } diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 72b19ee6cde..ad7bc5c194b 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4248,10 +4248,10 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; my $run_db = 'postgres'; - # Skip command-level tests for gzip if there is no support for it. + # Skip command-level tests for gzip/lz4 if they're not supported. if ($pgdump_runs{$run}->{compile_option} && - ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || - ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4)) + (($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) || + ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))) { note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support"; next; -- 2.34.1 --VbXIs2ae+j0NhctA Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-pg_dump-zstd-compression.patch" ^ permalink raw reply [nested|flat] 5+ messages in thread
* [PATCH 1/4] f!fixes for LZ4 @ 2023-02-24 20:07 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Justin Pryzby @ 2023-02-24 20:07 UTC (permalink / raw) --- src/bin/pg_dump/compress_gzip.c | 8 -------- src/bin/pg_dump/compress_io.h | 2 +- src/bin/pg_dump/compress_lz4.c | 12 ++++-------- src/bin/pg_dump/pg_dump.c | 2 -- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c index 0af65afeb4e..52f41c2e58c 100644 --- a/src/bin/pg_dump/compress_gzip.c +++ b/src/bin/pg_dump/compress_gzip.c @@ -123,17 +123,9 @@ WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs, gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1); gzipcs->outsize = ZLIB_OUT_SIZE; - /* - * A level of zero simply copies the input one block at the time. This - * is probably not what the user wanted when calling this interface. - */ - if (cs->compression_spec.level == 0) - pg_fatal("requested to compress the archive yet no level was specified"); - if (deflateInit(zp, cs->compression_spec.level) != Z_OK) pg_fatal("could not initialize compression library: %s", zp->msg); - /* Just be paranoid - maybe End is called after Start, with no Write */ zp->next_out = gzipcs->outbuf; zp->avail_out = gzipcs->outsize; } diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index bbde2693915..cdb15951ea9 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -172,7 +172,7 @@ struct CompressFileHandle extern CompressFileHandle *InitCompressFileHandle(const pg_compress_specification compression_spec); /* - * Initialize a compress file stream. Deffer the compression algorithm + * Initialize a compress file stream. Infer the compression algorithm * from 'path', either by examining its suffix or by appending the supported * suffixes in 'path'. */ diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index fe1014e6e77..7dacfeae469 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -105,12 +105,8 @@ EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs) LZ4CompressorState *LZ4cs; LZ4cs = (LZ4CompressorState *) cs->private_data; - if (LZ4cs) - { - pg_free(LZ4cs->outbuf); - pg_free(LZ4cs); - cs->private_data = NULL; - } + pg_free(LZ4cs->outbuf); + pg_free(LZ4cs); } @@ -161,8 +157,8 @@ typedef struct LZ4File } LZ4File; /* - * LZ4 equivalent to feof() or gzeof(). The end of file is reached if there - * is no decompressed output in the overflow buffer and the end of the file + * LZ4 equivalent to feof() or gzeof(). Return true iff there is no + * decompressed output in the overflow buffer and the end of the backing file * is reached. */ static int diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 24ba936332d..ce2242195f3 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -733,8 +733,6 @@ main(int argc, char **argv) #ifdef HAVE_LIBZ parse_compress_specification(PG_COMPRESSION_GZIP, NULL, &compression_spec); -#else - /* Nothing to do in the default case */ #endif } -- 2.34.1 --Fw50BJRL4GwAnGn1 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-f-fixes-for-LZ4-which-also-conflict-with-the-ZSTD-pa.patch" ^ permalink raw reply [nested|flat] 5+ messages in thread
end of thread, other threads:[~2023-02-24 20:07 UTC | newest] Thread overview: 5+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-09-13 10:12 [PATCH 3/6] Optimize allocations in bringetbitmap Tomas Vondra <[email protected]> 2023-02-24 20:07 [PATCH] f!fixes for LZ4 Justin Pryzby <[email protected]> 2023-02-24 20:07 [PATCH] f!fixes for LZ4 Justin Pryzby <[email protected]> 2023-02-24 20:07 [PATCH 1/4] f!fixes for LZ4 Justin Pryzby <[email protected]> 2023-02-24 20:07 [PATCH 1/4] f!fixes for LZ4 Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox