public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 3/8] Allow alternate compression methods for wal_compression 5+ messages / 5 participants [nested] [flat]
* [PATCH 3/8] Allow alternate compression methods for wal_compression @ 2021-02-27 04:03 Andrey Borodin <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Andrey Borodin @ 2021-02-27 04:03 UTC (permalink / raw) TODO: bump XLOG_PAGE_MAGIC --- src/backend/Makefile | 2 +- src/backend/access/transam/xlog.c | 10 ++++ src/backend/access/transam/xloginsert.c | 52 +++++++++++++++++++-- src/backend/access/transam/xlogreader.c | 62 ++++++++++++++++++++++++- src/backend/utils/misc/guc.c | 11 +++++ src/include/access/xlog.h | 1 + src/include/access/xlog_internal.h | 8 ++++ src/include/access/xlogreader.h | 1 + src/include/access/xlogrecord.h | 9 ++-- 9 files changed, 144 insertions(+), 12 deletions(-) diff --git a/src/backend/Makefile b/src/backend/Makefile index 0da848b1fd..3af216ddfc 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -48,7 +48,7 @@ OBJS = \ LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) # The backend doesn't need everything that's in LIBS, however -LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) +LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) ifeq ($(with_systemd),yes) LIBS += -lsystemd diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e3128564e1..0183589b4d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; +int wal_compression_method = WAL_COMPRESSION_PGLZ; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -180,6 +181,15 @@ const struct config_enum_entry recovery_target_action_options[] = { {NULL, 0, false} }; +/* Note that due to conditional compilation, offsets within the array are not static */ +const struct config_enum_entry wal_compression_options[] = { + {"pglz", WAL_COMPRESSION_PGLZ, false}, +#ifdef HAVE_LIBZ + {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif + {NULL, 0, false} +}; + /* * Statistics for current checkpoint are collected in this global struct. * Because only the checkpointer or a stand-alone backend can perform diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 7052dc245e..ee73bc3afd 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -33,6 +33,10 @@ #include "storage/proc.h" #include "utils/memutils.h" +#ifdef HAVE_LIBZ +#include <zlib.h> +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -113,7 +117,8 @@ static XLogRecData *XLogRecordAssemble(RmgrId rmid, uint8 info, XLogRecPtr RedoRecPtr, bool doPageWrites, XLogRecPtr *fpw_lsn, int *num_fpi); static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, - uint16 hole_length, char *dest, uint16 *dlen); + uint16 hole_length, char *dest, + uint16 *dlen, WalCompression compression); /* * Begin constructing a WAL record. This must be called before the @@ -630,11 +635,12 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, */ if (wal_compression) { + bimg.compression_method = wal_compression_method; is_compressed = XLogCompressBackupBlock(page, bimg.hole_offset, cbimg.hole_length, regbuf->compressed_page, - &compressed_len); + &compressed_len, bimg.compression_method); } /* @@ -827,7 +833,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, */ static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, - char *dest, uint16 *dlen) + char *dest, uint16 *dlen, WalCompression compression) { int32 orig_len = BLCKSZ - hole_length; int32 len; @@ -853,12 +859,48 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, else source = page; + switch (compression) + { + case WAL_COMPRESSION_PGLZ: + len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); + break; + +#ifdef HAVE_LIBZ + case WAL_COMPRESSION_ZLIB: + { + unsigned long len_l = PGLZ_MAX_BLCKSZ; + int ret = compress2((Bytef*)dest, &len_l, (Bytef*)source, orig_len, + Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + { + // XXX: using an interface other than compress() would allow giving a better error message + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("failed compressing zlib (%d)", ret))); + len_l = -1; + } + len = len_l; + break; + } +#endif + + default: + /* + * It should be impossible to get here for unsupported algorithms, + * which cannot be assigned if they're not enabled at compile time. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unknown compression method requested: %d(%s)", + compression, wal_compression_name(compression)))); + + } + /* - * We recheck the actual size even if pglz_compress() reports success and + * We recheck the actual size even if compression reports success and * see if the number of bytes saved by compression is larger than the * length of extra data needed for the compressed version of block image. */ - len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); if (len >= 0 && len + extra_bytes < orig_len) { diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 42738eb940..143df55fcb 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -33,6 +33,10 @@ #include "utils/memutils.h" #endif +#ifdef HAVE_LIBZ +#include <zlib.h> +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1286,6 +1290,7 @@ DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, char **errormsg) { COPY_HEADER_FIELD(&blk->bimg_len, sizeof(uint16)); COPY_HEADER_FIELD(&blk->hole_offset, sizeof(uint16)); + COPY_HEADER_FIELD(&blk->compression_method, sizeof(uint8)); COPY_HEADER_FIELD(&blk->bimg_info, sizeof(uint8)); blk->apply_image = ((blk->bimg_info & BKPIMAGE_APPLY) != 0); @@ -1535,6 +1540,29 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) } } +/* + * Return a statically allocated string associated with the given compression + * method. This is similar to the guc, but isn't subject to conditional + * compilation. + */ +const char * +wal_compression_name(WalCompression compression) +{ + /* + * This could index into the guc array, except that it's compiled + * conditionally and unsupported methods are elided. + */ + switch (compression) + { + case WAL_COMPRESSION_PGLZ: + return "pglz"; + case WAL_COMPRESSION_ZLIB: + return "zlib"; + default: + return "???"; + } +} + /* * Restore a full-page image from a backup block attached to an XLOG record. * @@ -1558,8 +1586,38 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) if (bkpb->bimg_info & BKPIMAGE_IS_COMPRESSED) { /* If a backup block image is compressed, decompress it */ - if (pglz_decompress(ptr, bkpb->bimg_len, tmp.data, - BLCKSZ - bkpb->hole_length, true) < 0) + int32 decomp_result = -1; + switch (bkpb->compression_method) + { + case WAL_COMPRESSION_PGLZ: + decomp_result = pglz_decompress(ptr, bkpb->bimg_len, tmp.data, + BLCKSZ - bkpb->hole_length, true); + break; + +#ifdef HAVE_LIBZ + case WAL_COMPRESSION_ZLIB: + { + unsigned long decomp_result_l = 0; + decomp_result_l = BLCKSZ - bkpb->hole_length; + if (uncompress((Bytef*)tmp.data, &decomp_result_l, (Bytef*)ptr, bkpb->bimg_len) == Z_OK) + decomp_result = decomp_result_l; + else + decomp_result = -1; + break; + } +#endif + + default: + report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", + (uint32) (record->ReadRecPtr >> 32), + (uint32) record->ReadRecPtr, + block_id, + bkpb->compression_method, + wal_compression_name(bkpb->compression_method)); + return false; + } + + if (decomp_result < 0) { report_invalid_record(record, "invalid compressed image at %X/%X, block %d", LSN_FORMAT_ARGS(record->ReadRecPtr), diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 855076b1fd..8084027465 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -508,6 +508,7 @@ extern const struct config_enum_entry archive_mode_options[]; extern const struct config_enum_entry recovery_target_action_options[]; extern const struct config_enum_entry sync_method_options[]; extern const struct config_enum_entry dynamic_shared_memory_options[]; +extern const struct config_enum_entry wal_compression_options[]; /* * GUC option variables that are exported from this module @@ -4721,6 +4722,16 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"wal_compression_method", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Set the method used to compress full page images in the WAL."), + NULL + }, + &wal_compression_method, + WAL_COMPRESSION_PGLZ, wal_compression_options, + NULL, NULL, NULL + }, + { {"dynamic_shared_memory_type", PGC_POSTMASTER, RESOURCES_MEM, gettext_noop("Selects the dynamic shared memory implementation used."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 6d384d3ce6..fa2e5c611f 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -117,6 +117,7 @@ extern bool EnableHotStandby; extern bool fullPageWrites; extern bool wal_log_hints; extern bool wal_compression; +extern int wal_compression_method; extern bool wal_init_zero; extern bool wal_recycle; extern bool *wal_consistency_checking; diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b23e286406..b80759ed45 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -324,4 +324,12 @@ extern bool InArchiveRecovery; extern bool StandbyMode; extern char *recoveryRestoreCommand; +typedef enum WalCompression +{ + WAL_COMPRESSION_PGLZ = 0, + WAL_COMPRESSION_ZLIB = 1, +} WalCompression; + +extern const char *wal_compression_name(WalCompression compression); + #endif /* XLOG_INTERNAL_H */ diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 21d200d3df..3d19c315d7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -133,6 +133,7 @@ typedef struct bool apply_image; /* has image that should be restored */ char *bkp_image; uint16 hole_offset; + uint8 compression_method; uint16 hole_length; uint16 bimg_len; uint8 bimg_info; diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 80c92a2498..0d4c212f15 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -114,7 +114,7 @@ typedef struct XLogRecordBlockHeader * present is (BLCKSZ - <length of "hole" bytes>). * * Additionally, when wal_compression is enabled, we will try to compress full - * page images using the PGLZ compression algorithm, after removing the "hole". + * page images, after removing the "hole". * This can reduce the WAL volume, but at some extra cost of CPU spent * on the compression during WAL logging. In this case, since the "hole" * length cannot be calculated by subtracting the number of page image bytes @@ -129,9 +129,10 @@ typedef struct XLogRecordBlockHeader */ typedef struct XLogRecordBlockImageHeader { - uint16 length; /* number of page image bytes */ - uint16 hole_offset; /* number of bytes before "hole" */ - uint8 bimg_info; /* flag bits, see below */ + uint16 length; /* number of page image bytes */ + uint16 hole_offset; /* number of bytes before "hole" */ + uint8 compression_method; /* compression method used for image */ + uint8 bimg_info; /* flag bits, see below */ /* * If BKPIMAGE_HAS_HOLE and BKPIMAGE_IS_COMPRESSED, an -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-wal_compression_method-default-to-zlib.patch" ^ permalink raw reply [nested|flat] 5+ messages in thread
* Patch: Code comments: why some text-handling functions are leakproof @ 2022-01-11 07:07 Gurjeet Singh <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Gurjeet Singh @ 2022-01-11 07:07 UTC (permalink / raw) To: pgsql-hackers Please see attached a small patch to document why some text-processing functions are marked as leakproof, while some others are not. This is more or less a verbatim copy of Tom's comment in email thread at [1]. I could not find an appropriate spot to place these comments, so I placed them on bttextcmp() function, The only other place that I could see we can place these comments is in the file src/backend/optimizer/README, because there is some consideration given to leakproof functions in optimizer docs. But these comments seem quite out of place in optimizer docs. [1]: https://www.postgresql.org/message-id/flat/673096.1630006990%40sss.pgh.pa.us#cd378cba4b990fda070c6fa... Best regards, -- Gurjeet Singh http://gurjeet.singh.im/ Attachments: [application/octet-stream] leakproof_comments.patch (1.3K, ../../CABwTF4Wjp3q=dv6wGgqmFxuE7ZJmas=R_BjJrQHnJwkHzZmd9g@mail.gmail.com/3-leakproof_comments.patch) download | inline diff: diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index bd3091bbfb..4e464ad09d 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -1974,6 +1974,25 @@ text_starts_with(PG_FUNCTION_ARGS) PG_RETURN_BOOL(result); } +/* + * Generally speaking, we should be resistant to marking anything leakproof + * unless it has a very small code footprint that can be easily audited. In + * particular, anything that shares a lot of infrastructure with not-leakproof + * functions seems quite hazardous. Even if we go through the code and + * convince ourselves that it's OK today, innocent changes to the shared + * infrastructure could break the leakproofness tomorrow. + * + * The function bttextcmp() and its cohorts are marked as leakproof, but the + * user-visible string processing functions, like upper(), are not, because: + * + * 1. The query-optimization usefulness of having those be leakproof + * is extremely high. + * + * 2. btree comparison functions should really not have any user-reachable + * failure modes (which comes close to being the definition of leakproof). If one + * did, that would mean there were legal values of the type that couldn't be put + * into a btree index. + */ Datum bttextcmp(PG_FUNCTION_ARGS) { ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Patch: Code comments: why some text-handling functions are leakproof @ 2022-01-13 20:26 Robert Haas <[email protected]> parent: Gurjeet Singh <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Robert Haas @ 2022-01-13 20:26 UTC (permalink / raw) To: Gurjeet Singh <[email protected]>; +Cc: pgsql-hackers On Tue, Jan 11, 2022 at 2:07 AM Gurjeet Singh <[email protected]> wrote: > Please see attached a small patch to document why some text-processing functions are marked as leakproof, while some others are not. > > This is more or less a verbatim copy of Tom's comment in email thread at [1]. > > I could not find an appropriate spot to place these comments, so I placed them on bttextcmp() function, The only other place that I could see we can place these comments is in the file src/backend/optimizer/README, because there is some consideration given to leakproof functions in optimizer docs. But these comments seem quite out of place in optimizer docs. It doesn't seem particularly likely that someone who is thinking about changing this in the future would notice the comment in the place where you propose to put it, nor that they would read the optimizer README. Furthermore, I don't know that everyone agrees with Tom about this. I do agree that it's more important to mark relational operators leakproof than other things, and I also agree that conservatism is warranted. But that does not mean that someone could not make a compelling argument for marking other functions leakproof. I think we will be better off leaving this alone. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Patch: Code comments: why some text-handling functions are leakproof @ 2022-02-28 22:02 Tom Lane <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Tom Lane @ 2022-02-28 22:02 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: Gurjeet Singh <[email protected]>; [email protected] Robert Haas <[email protected]> writes: > On Tue, Jan 11, 2022 at 2:07 AM Gurjeet Singh <[email protected]> wrote: >> This is more or less a verbatim copy of Tom's comment in email thread at [1]. >> >> I could not find an appropriate spot to place these comments, so I placed them on bttextcmp() function, The only other place that I could see we can place these comments is in the file src/backend/optimizer/README, because there is some consideration given to leakproof functions in optimizer docs. But these comments seem quite out of place in optimizer docs. > It doesn't seem particularly likely that someone who is thinking about > changing this in the future would notice the comment in the place > where you propose to put it, nor that they would read the optimizer > README. Agreed. I think if we wanted to make an upgrade in the way function leakproofness is documented, we ought to add a <sect1> about it in xfunc.sgml, adjacent to the one about function volatility categories. This could perhaps consolidate some of the existing documentation mentions of leakproofness, as well as adding text similar to what Gurjeet suggests. > Furthermore, I don't know that everyone agrees with Tom about this. I > do agree that it's more important to mark relational operators > leakproof than other things, and I also agree that conservatism is > warranted. But that does not mean that someone could not make a > compelling argument for marking other functions leakproof. ISTM the proposed text does a reasonable job of explaining why we made the decisions currently embedded in pg_proc.proleakproof. If we make some other decisions in future, updating the rationale in the docs would be an appropriate part of that. regards, tom lane ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Patch: Code comments: why some text-handling functions are leakproof @ 2022-03-28 18:55 Greg Stark <[email protected]> parent: Tom Lane <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Greg Stark @ 2022-03-28 18:55 UTC (permalink / raw) To: Tom Lane <[email protected]>; +Cc: Robert Haas <[email protected]>; Gurjeet Singh <[email protected]>; PostgreSQL Hackers <[email protected]> I'm going to mark this returned with feedback. If you have a chance to update the patch moving the documentation to xfunc.sgml the way Tom describes make sure to create a new commitfest entry. I would suggest submitting the patch as a followup on this thread so when it's added to the commitfest it links to this whole discussion. On Mon, 28 Feb 2022 at 17:12, Tom Lane <[email protected]> wrote: > > Robert Haas <[email protected]> writes: > > On Tue, Jan 11, 2022 at 2:07 AM Gurjeet Singh <[email protected]> wrote: > >> This is more or less a verbatim copy of Tom's comment in email thread at [1]. > >> > >> I could not find an appropriate spot to place these comments, so I placed them on bttextcmp() function, The only other place that I could see we can place these comments is in the file src/backend/optimizer/README, because there is some consideration given to leakproof functions in optimizer docs. But these comments seem quite out of place in optimizer docs. > > > It doesn't seem particularly likely that someone who is thinking about > > changing this in the future would notice the comment in the place > > where you propose to put it, nor that they would read the optimizer > > README. > > Agreed. I think if we wanted to make an upgrade in the way function > leakproofness is documented, we ought to add a <sect1> about it in > xfunc.sgml, adjacent to the one about function volatility categories. > This could perhaps consolidate some of the existing documentation mentions > of leakproofness, as well as adding text similar to what Gurjeet suggests. > > > Furthermore, I don't know that everyone agrees with Tom about this. I > > do agree that it's more important to mark relational operators > > leakproof than other things, and I also agree that conservatism is > > warranted. But that does not mean that someone could not make a > > compelling argument for marking other functions leakproof. > > ISTM the proposed text does a reasonable job of explaining why > we made the decisions currently embedded in pg_proc.proleakproof. > If we make some other decisions in future, updating the rationale > in the docs would be an appropriate part of that. > > regards, tom lane > > -- greg ^ permalink raw reply [nested|flat] 5+ messages in thread
end of thread, other threads:[~2022-03-28 18:55 UTC | newest] Thread overview: 5+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-27 04:03 [PATCH 3/8] Allow alternate compression methods for wal_compression Andrey Borodin <[email protected]> 2022-01-11 07:07 Patch: Code comments: why some text-handling functions are leakproof Gurjeet Singh <[email protected]> 2022-01-13 20:26 ` Re: Patch: Code comments: why some text-handling functions are leakproof Robert Haas <[email protected]> 2022-02-28 22:02 ` Re: Patch: Code comments: why some text-handling functions are leakproof Tom Lane <[email protected]> 2022-03-28 18:55 ` Re: Patch: Code comments: why some text-handling functions are leakproof Greg Stark <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox