agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 1/5] Built-in compression method 9+ messages / 2 participants [nested] [flat]
* [PATCH 1/5] Built-in compression method @ 2021-03-05 04:03 Dilip Kumar <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Dilip Kumar @ 2021-03-05 04:03 UTC (permalink / raw) Add syntax allowing a compression method to be specified. As of now there is only 2 option for build-in compression method (pglz, lz4) which can be set while creating a table or adding a new column. No option for altering the compression method for an existing column. Dilip Kumar based on the patches from Ildus Kurbangaliev. Design input from Robert Haas and Tomas Vondra. Reviewed by Robert Haas, Tomas Vondra, Alexander Korotkov and Justin Pryzby Discussions: https://www.postgresql.org/message-id/[email protected] https://www.postgresql.org/message-id/CA%2BTgmoaKDW1Oi9V%3Djc9hOGyf77NbkNEABuqgHD1Cq%3D%3D1QsOcxg%40... https://www.postgresql.org/message-id/CA%2BTgmobSDVgUage9qQ5P_%3DF_9jaMkCgyKxUQGtFQU7oN4kX-AA%40mail... https://www.postgresql.org/message-id/20201005160355.byp74sh3ejsv7wrj%40development https://www.postgresql.org/message-id/CAFiTN-tzTTT2oqWdRGLv1dvvS5MC1W%2BLE%2B3bqWPJUZj4GnHOJg%40mail... --- configure | 118 +++++++ configure.ac | 18 + doc/src/sgml/catalogs.sgml | 10 + doc/src/sgml/ref/create_table.sgml | 33 +- doc/src/sgml/ref/psql-ref.sgml | 11 + src/backend/access/brin/brin_tuple.c | 5 +- src/backend/access/common/Makefile | 1 + src/backend/access/common/detoast.c | 71 ++-- src/backend/access/common/indextuple.c | 3 +- src/backend/access/common/toast_compression.c | 308 ++++++++++++++++++ src/backend/access/common/toast_internals.c | 54 ++- src/backend/access/common/tupdesc.c | 8 + src/backend/access/table/toast_helper.c | 5 +- src/backend/bootstrap/bootstrap.c | 5 + src/backend/catalog/genbki.pl | 3 + src/backend/catalog/heap.c | 4 + src/backend/catalog/index.c | 1 + src/backend/catalog/toasting.c | 6 + src/backend/commands/tablecmds.c | 110 +++++++ src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/nodeFuncs.c | 2 + src/backend/nodes/outfuncs.c | 1 + src/backend/parser/gram.y | 26 +- src/backend/parser/parse_utilcmd.c | 9 + src/backend/utils/adt/varlena.c | 41 +++ src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_dump.c | 39 +++ src/bin/pg_dump/pg_dump.h | 1 + src/bin/pg_dump/t/002_pg_dump.pl | 12 +- src/bin/psql/describe.c | 31 +- src/bin/psql/help.c | 2 + src/bin/psql/settings.h | 1 + src/bin/psql/startup.c | 10 + src/include/access/detoast.h | 8 + src/include/access/toast_compression.h | 119 +++++++ src/include/access/toast_helper.h | 1 + src/include/access/toast_internals.h | 20 +- src/include/catalog/pg_attribute.h | 8 +- src/include/catalog/pg_proc.dat | 4 + src/include/nodes/parsenodes.h | 2 + src/include/parser/kwlist.h | 1 + src/include/pg_config.h.in | 3 + src/include/postgres.h | 14 +- src/test/regress/expected/compression.out | 247 ++++++++++++++ src/test/regress/expected/compression_1.out | 240 ++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/pg_regress_main.c | 4 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/compression.sql | 102 ++++++ src/tools/msvc/Solution.pm | 1 + src/tools/pgindent/typedefs.list | 1 + 52 files changed, 1645 insertions(+), 85 deletions(-) create mode 100644 src/backend/access/common/toast_compression.c create mode 100644 src/include/access/toast_compression.h create mode 100644 src/test/regress/expected/compression.out create mode 100644 src/test/regress/expected/compression_1.out create mode 100644 src/test/regress/sql/compression.sql diff --git a/configure b/configure index fad817bb38..761a27965d 100755 --- a/configure +++ b/configure @@ -699,6 +699,7 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +865,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -1569,6 +1571,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -8563,6 +8566,41 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + # # Assignments # @@ -12110,6 +12148,56 @@ fi fi +if test "$with_lz4" = yes ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4_compress in -llz4" >&5 +$as_echo_n "checking for LZ4_compress in -llz4... " >&6; } +if ${ac_cv_lib_lz4_LZ4_compress+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-llz4 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char LZ4_compress (); +int +main () +{ +return LZ4_compress (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_lz4_LZ4_compress=yes +else + ac_cv_lib_lz4_LZ4_compress=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lz4_LZ4_compress" >&5 +$as_echo "$ac_cv_lib_lz4_LZ4_compress" >&6; } +if test "x$ac_cv_lib_lz4_LZ4_compress" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBLZ4 1 +_ACEOF + + LIBS="-llz4 $LIBS" + +else + as_fn_error $? "library 'lz4' is required for LZ4 support" "$LINENO" 5 +fi + +fi + if test "$enable_spinlocks" = yes; then $as_echo "#define HAVE_SPINLOCKS 1" >>confdefs.h @@ -13376,6 +13464,36 @@ Use --without-zlib to disable zlib support." "$LINENO" 5 fi +fi + +if test "$with_lz4" = yes ; then + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + fi if test "$with_gssapi" = yes ; then diff --git a/configure.ac b/configure.ac index 0ed53571dd..616ce5e1a2 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,15 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + # # Assignments # @@ -1174,6 +1183,10 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes ; then + AC_CHECK_LIB(lz4, LZ4_compress, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) +fi + if test "$enable_spinlocks" = yes; then AC_DEFINE(HAVE_SPINLOCKS, 1, [Define to 1 if you have spinlocks.]) else @@ -1407,6 +1420,11 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes ; then + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index b1de6d0674..5fdc80ff3d 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1355,6 +1355,16 @@ </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>attcompression</structfield> <type>char</type> + </para> + <para> + The current compression method of the column. Must be <literal>InvalidCompressionMethod</literal> + if and only if typstorage is 'plain' or 'external'. + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>attacl</structfield> <type>aclitem[]</type> diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 3b2b227683..7c8fc77983 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation <refsynopsisdiv> <synopsis> CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] <replaceable class="parameter">table_name</replaceable> ( [ - { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ] + { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ COMPRESSION <replaceable>compression_method</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ] | <replaceable>table_constraint</replaceable> | LIKE <replaceable>source_table</replaceable> [ <replaceable>like_option</replaceable> ... ] } [, ... ] @@ -288,6 +288,26 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </listitem> </varlistentry> + <varlistentry> + <term><literal>COMPRESSION <replaceable class="parameter">compression_method</replaceable></literal></term> + <listitem> + <para> + The <literal>COMPRESSION</literal> clause sets the compression method + for a column. Compression is supported only for variable-width data + types, and is used only for columns whose storage type is main or + extended. (See <xref linkend="sql-altertable"/> for information on + column storage types.) Setting this property for a partitioned table + has no direct effect, because such tables have no storage of their own, + but the configured value is inherited by newly-created partitions. + The supported compression methods are <literal>pglz</literal> and + <literal>lz4</literal>. <literal>lz4</literal> is available only if + <literal>--with-lz4</literal> was used when building + <productname>PostgreSQL</productname>. The default + is <literal>pglz</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><literal>INHERITS ( <replaceable>parent_table</replaceable> [, ... ] )</literal></term> <listitem> @@ -605,6 +625,17 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </listitem> </varlistentry> + <varlistentry> + <term><literal>INCLUDING COMPRESSION</literal></term> + <listitem> + <para> + Compression method of the columns will be copied. The default + behavior is to exclude compression methods, resulting in columns + having the default compression method. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><literal>INCLUDING CONSTRAINTS</literal></term> <listitem> diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 13c1edfa4d..01ec9b8b0a 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -3863,6 +3863,17 @@ bar </listitem> </varlistentry> + <varlistentry> + <term><varname>HIDE_TOAST_COMPRESSION</varname></term> + <listitem> + <para> + If this variable is set to <literal>true</literal>, column + compression method details are not displayed. This is mainly + useful for regression tests. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><varname>HIDE_TABLEAM</varname></term> <listitem> diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index a7eb1c9473..0ab5712c71 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -213,7 +213,10 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, (atttype->typstorage == TYPSTORAGE_EXTENDED || atttype->typstorage == TYPSTORAGE_MAIN)) { - Datum cvalue = toast_compress_datum(value); + Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc, + keyno); + Datum cvalue = toast_compress_datum(value, + att->attcompression); if (DatumGetPointer(cvalue) != NULL) { diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index 5a007d63f1..b9aff0ccfd 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -25,6 +25,7 @@ OBJS = \ scankey.o \ session.o \ syncscan.o \ + toast_compression.o \ toast_internals.o \ tupconvert.o \ tupdesc.o diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index d1cdbaf648..031d20d5fc 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -13,6 +13,7 @@ #include "postgres.h" +#include "access/toast_compression.h" #include "access/detoast.h" #include "access/table.h" #include "access/tableam.h" @@ -456,6 +457,37 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, return result; } +/* ---------- + * toast_get_compression_method + * + * Returns compression method for the compressed varlena. If it is not + * compressed then returns InvalidCompressionMethod. + */ +char +toast_get_compression_method(struct varlena *attr) +{ + if (VARATT_IS_EXTERNAL_ONDISK(attr)) + { + struct varatt_external toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + /* fast path for non-compressed external datums */ + if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + return InvalidCompressionMethod; + + /* + * Fetch just enough of the value to examine the compression header, + * so that we can find out the compression method. + */ + attr = toast_fetch_datum_slice(attr, 0, VARHDRSZ); + } + else if (!VARATT_IS_COMPRESSED(attr)) + return InvalidCompressionMethod; + + return CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr)); +} + /* ---------- * toast_decompress_datum - * @@ -464,21 +496,18 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, static struct varlena * toast_decompress_datum(struct varlena *attr) { - struct varlena *result; + const CompressionRoutine *cmroutine; Assert(VARATT_IS_COMPRESSED(attr)); - result = (struct varlena *) - palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); - SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); - - if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), - TOAST_COMPRESS_SIZE(attr), - VARDATA(result), - TOAST_COMPRESS_RAWSIZE(attr), true) < 0) - elog(ERROR, "compressed data is corrupted"); + /* + * Get compression handler routines, using the compression id stored in the + * toast header. + */ + cmroutine = GetCompressionRoutines( + CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr))); - return result; + return cmroutine->datum_decompress(attr); } @@ -492,22 +521,18 @@ toast_decompress_datum(struct varlena *attr) static struct varlena * toast_decompress_datum_slice(struct varlena *attr, int32 slicelength) { - struct varlena *result; - int32 rawsize; + const CompressionRoutine *cmroutine; Assert(VARATT_IS_COMPRESSED(attr)); - result = (struct varlena *) palloc(slicelength + VARHDRSZ); - - rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), - VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, - VARDATA(result), - slicelength, false); - if (rawsize < 0) - elog(ERROR, "compressed data is corrupted"); + /* + * Get compression handler routines, using the compression id stored in the + * toast header. + */ + cmroutine = GetCompressionRoutines( + CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr))); - SET_VARSIZE(result, rawsize + VARHDRSZ); - return result; + return cmroutine->datum_decompress_slice(attr, slicelength); } /* ---------- diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index b72a138497..1f6b7b77d4 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -103,7 +103,8 @@ index_form_tuple(TupleDesc tupleDescriptor, (att->attstorage == TYPSTORAGE_EXTENDED || att->attstorage == TYPSTORAGE_MAIN)) { - Datum cvalue = toast_compress_datum(untoasted_values[i]); + Datum cvalue = toast_compress_datum(untoasted_values[i], + att->attcompression); if (DatumGetPointer(cvalue) != NULL) { diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c new file mode 100644 index 0000000000..3463b42438 --- /dev/null +++ b/src/backend/access/common/toast_compression.c @@ -0,0 +1,308 @@ +/*------------------------------------------------------------------------- + * + * toast_compression.c + * Functions for toast compression. + * + * Copyright (c) 2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/common/toast_compression.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#ifdef HAVE_LIBLZ4 +#include <lz4.h> +#endif + +#include "access/toast_compression.h" +#include "common/pg_lzcompress.h" +#include "fmgr.h" +#include "utils/builtins.h" + +static struct varlena *pglz_cmcompress(const struct varlena *value); +static struct varlena *pglz_cmdecompress(const struct varlena *value); +static struct varlena *pglz_cmdecompress_slice(const struct varlena *value, + int32 slicelength); +static struct varlena *lz4_cmcompress(const struct varlena *value); +static struct varlena *lz4_cmdecompress(const struct varlena *value); +static struct varlena *lz4_cmdecompress_slice(const struct varlena *value, + int32 slicelength); + +#define NO_LZ4_SUPPORT() \ + ereport(ERROR, \ + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ + errmsg("unsupported LZ4 compression method"), \ + errdetail("This functionality requires the server to be built with lz4 support."), \ + errhint("You need to rebuild PostgreSQL using --with-lz4."))) + +/* handler routines for pglz and lz4 built-in compression methods */ +const CompressionRoutine toast_compression[] = +{ + { + .cmname = "pglz", + .datum_compress = pglz_cmcompress, + .datum_decompress = pglz_cmdecompress, + .datum_decompress_slice = pglz_cmdecompress_slice + }, + { + .cmname = "lz4", + .datum_compress = lz4_cmcompress, + .datum_decompress = lz4_cmdecompress, + .datum_decompress_slice = lz4_cmdecompress_slice + } +}; + +/* + * pglz_cmcompress - compression routine for pglz compression method + * + * Compresses source into dest using the default strategy. Returns the + * compressed varlena, or NULL if compression fails. + */ +static struct varlena * +pglz_cmcompress(const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + + /* + * No point in wasting a palloc cycle if value size is outside the allowed + * range for compression. + */ + if (valsize < PGLZ_strategy_default->min_input_size || + valsize > PGLZ_strategy_default->max_input_size) + return NULL; + + /* + * Figure out the maximum possible size of the pglz output, add the bytes + * that will be needed for varlena overhead, and allocate that amount. + */ + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_COMPRESS); + + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_COMPRESS, + NULL); + if (len < 0) + { + pfree(tmp); + return NULL; + } + + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS); + + return tmp; +} + +/* + * pglz_cmdecompress - decompression routine for pglz compression method + * + * Returns the decompressed varlena. + */ +static struct varlena * +pglz_cmdecompress(const struct varlena *value) +{ + struct varlena *result; + int32 rawsize; + + /* allocate memory for the uncompressed data */ + result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); + + /* decompress the data */ + rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS, + VARSIZE(value) - VARHDRSZ_COMPRESS, + VARDATA(result), + VARRAWSIZE_4B_C(value), true); + if (rawsize < 0) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed pglz data is corrupt"))); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +} + +/* + * pglz_decompress - slice decompression routine for pglz compression method + * + * Decompresses part of the data. Returns the decompressed varlena. + */ +static struct varlena * +pglz_cmdecompress_slice(const struct varlena *value, + int32 slicelength) +{ + struct varlena *result; + int32 rawsize; + + /* allocate memory for the uncompressed data */ + result = (struct varlena *) palloc(slicelength + VARHDRSZ); + + /* decompress the data */ + rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS, + VARSIZE(value) - VARHDRSZ_COMPRESS, + VARDATA(result), + slicelength, false); + if (rawsize < 0) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed pglz data is corrupt"))); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +} + +/* + * lz4_cmcompress - compression routine for lz4 compression method + * + * Compresses source into dest using the LZ4 defaults. Returns the + * compressed varlena, or NULL if compression fails. + */ +static struct varlena * +lz4_cmcompress(const struct varlena *value) +{ +#ifndef HAVE_LIBLZ4 + NO_LZ4_SUPPORT(); +#else + int32 valsize; + int32 len; + int32 max_size; + struct varlena *tmp = NULL; + + valsize = VARSIZE_ANY_EXHDR(value); + + /* + * Figure out the maximum possible size of the LZ4 output, add the bytes + * that will be needed for varlena overhead, and allocate that amount. + */ + max_size = LZ4_compressBound(valsize); + tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESS); + + len = LZ4_compress_default(VARDATA_ANY(value), + (char *) tmp + VARHDRSZ_COMPRESS, + valsize, max_size); + if (len <= 0) + elog(ERROR, "could not compress data with lz4"); + + /* data is incompressible so just free the memory and return NULL */ + if (len > valsize) + { + pfree(tmp); + return NULL; + } + + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS); + + return tmp; +#endif +} + +/* + * lz4_cmdecompress - decompression routine for lz4 compression method + * + * Returns the decompressed varlena. + */ +static struct varlena * +lz4_cmdecompress(const struct varlena *value) +{ +#ifndef HAVE_LIBLZ4 + NO_LZ4_SUPPORT(); +#else + int32 rawsize; + struct varlena *result; + + /* allocate memory for the uncompressed data */ + result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); + + /* decompress the data */ + rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESS, + VARDATA(result), + VARSIZE(value) - VARHDRSZ_COMPRESS, + VARRAWSIZE_4B_C(value)); + if (rawsize < 0) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed lz4 data is corrupt"))); + + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +#endif +} + +/* + * lz4_cmdecompress_slice - slice decompression routine for lz4 compression + * + * Decompresses part of the data. Returns the decompressed varlena. + */ +static struct varlena * +lz4_cmdecompress_slice(const struct varlena *value, int32 slicelength) +{ +#ifndef HAVE_LIBLZ4 + NO_LZ4_SUPPORT(); +#elif LZ4_VERSION_NUMBER < 10803 + return lz4_cmdecompress(value); +#else + int32 rawsize; + struct varlena *result; + + /* allocate memory for the uncompressed data */ + result = (struct varlena *) palloc(slicelength + VARHDRSZ); + + /* decompress the data */ + rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESS, + VARDATA(result), + VARSIZE(value) - VARHDRSZ_COMPRESS, + slicelength, + slicelength); + if (rawsize < 0) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("compressed lz4 data is corrupt"))); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +#endif +} + +/* + * CompressionNameToMethod - Get compression method from compression name + * + * Search in the available built-in methods. If the compression not found + * in the built-in methods then return InvalidCompressionMethod. + */ +char +CompressionNameToMethod(char *compression) +{ + if (strcmp(toast_compression[PGLZ_COMPRESSION_ID].cmname, + compression) == 0) + return PGLZ_COMPRESSION; + else if (strcmp(toast_compression[LZ4_COMPRESSION_ID].cmname, + compression) == 0) + { +#ifndef HAVE_LIBLZ4 + NO_LZ4_SUPPORT(); +#endif + return LZ4_COMPRESSION; + } + + return InvalidCompressionMethod; +} + +/* + * GetCompressionRoutines - Get compression handler routines + */ +const CompressionRoutine* +GetCompressionRoutines(char method) +{ + return &toast_compression[CompressionMethodToId(method)]; +} diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 9b9da0f41b..69dd9492f6 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -44,46 +44,42 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); * ---------- */ Datum -toast_compress_datum(Datum value) +toast_compress_datum(Datum value, char cmethod) { - struct varlena *tmp; - int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); - int32 len; + struct varlena *tmp = NULL; + int32 valsize; + const CompressionRoutine *cmroutine = NULL; Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value))); Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value))); - /* - * No point in wasting a palloc cycle if value size is out of the allowed - * range for compression - */ - if (valsize < PGLZ_strategy_default->min_input_size || - valsize > PGLZ_strategy_default->max_input_size) - return PointerGetDatum(NULL); + Assert(CompressionMethodIsValid(cmethod)); - tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + - TOAST_COMPRESS_HDRSZ); + /* get the handler routines for the compression method */ + cmroutine = GetCompressionRoutines(cmethod); + + /* call the actual compression function */ + tmp = cmroutine->datum_compress((const struct varlena *) value); + if (!tmp) + return PointerGetDatum(NULL); /* - * We recheck the actual size even if pglz_compress() reports success, - * because it might be satisfied with having saved as little as one byte - * in the compressed data --- which could turn into a net loss once you - * consider header and alignment padding. Worst case, the compressed - * format might require three padding bytes (plus header, which is - * included in VARSIZE(tmp)), whereas the uncompressed format would take - * only one header byte and no padding if the value is short enough. So - * we insist on a savings of more than 2 bytes to ensure we have a gain. + * We recheck the actual size even if compression reports success, because + * it might be satisfied with having saved as little as one byte in the + * compressed data --- which could turn into a net loss once you consider + * header and alignment padding. Worst case, the compressed format might + * require three padding bytes (plus header, which is included in + * VARSIZE(tmp)), whereas the uncompressed format would take only one + * header byte and no padding if the value is short enough. So we insist + * on a savings of more than 2 bytes to ensure we have a gain. */ - len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), - valsize, - TOAST_COMPRESS_RAWDATA(tmp), - PGLZ_strategy_default); - if (len >= 0 && - len + TOAST_COMPRESS_HDRSZ < valsize - 2) + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + + if (VARSIZE(tmp) < valsize - 2) { - TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize); - SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ); /* successful compression */ + TOAST_COMPRESS_SET_SIZE_AND_METHOD(tmp, valsize, + CompressionMethodToId(cmethod)); return PointerGetDatum(tmp); } else diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 902f59440c..503d64df38 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "access/toast_compression.h" #include "access/htup_details.h" #include "access/tupdesc_details.h" #include "catalog/pg_collation.h" @@ -470,6 +471,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (attr1->attcollation != attr2->attcollation) return false; + if (attr1->attcompression != attr2->attcompression) + return false; /* attacl, attoptions and attfdwoptions are not even present... */ } @@ -664,6 +667,11 @@ TupleDescInitEntry(TupleDesc desc, att->attstorage = typeForm->typstorage; att->attcollation = typeForm->typcollation; + if (IsStorageCompressible(typeForm->typstorage)) + att->attcompression = DefaultCompressionMethod; + else + att->attcompression = InvalidCompressionMethod; + ReleaseSysCache(tuple); } diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index fb36151ce5..53f78f9c3e 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -54,6 +54,7 @@ toast_tuple_init(ToastTupleContext *ttc) ttc->ttc_attr[i].tai_colflags = 0; ttc->ttc_attr[i].tai_oldexternal = NULL; + ttc->ttc_attr[i].tai_compression = att->attcompression; if (ttc->ttc_oldvalues != NULL) { @@ -226,9 +227,11 @@ void toast_tuple_try_compression(ToastTupleContext *ttc, int attribute) { Datum *value = &ttc->ttc_values[attribute]; - Datum new_value = toast_compress_datum(*value); + Datum new_value; ToastAttrInfo *attr = &ttc->ttc_attr[attribute]; + new_value = toast_compress_datum(*value, attr->tai_compression); + if (DatumGetPointer(new_value) != NULL) { /* successful compression */ diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 6f615e6622..45e1cfa56c 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -17,6 +17,7 @@ #include <unistd.h> #include <signal.h> +#include "access/toast_compression.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" @@ -731,6 +732,10 @@ DefineAttr(char *name, char *type, int attnum, int nullness) attrtypes[attnum]->attcacheoff = -1; attrtypes[attnum]->atttypmod = -1; attrtypes[attnum]->attislocal = true; + if (IsStorageCompressible(attrtypes[attnum]->attstorage)) + attrtypes[attnum]->attcompression = DefaultCompressionMethod; + else + attrtypes[attnum]->attcompression = InvalidCompressionMethod; if (nullness == BOOTCOL_NULL_FORCE_NOT_NULL) { diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index b159958112..9586c29ad0 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -906,6 +906,9 @@ sub morph_row_for_pgattr $row->{attcollation} = $type->{typcollation} ne '0' ? $C_COLLATION_OID : 0; + $row->{attcompression} = + $type->{typstorage} ne 'p' && $type->{typstorage} ne 'e' ? 'p' : '\0'; + if (defined $attr->{forcenotnull}) { $row->{attnotnull} = 't'; diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 9abc4a1f55..d8b71e5bc3 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -29,6 +29,7 @@ */ #include "postgres.h" +#include "access/toast_compression.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/multixact.h" @@ -789,6 +790,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, slot[slotCount]->tts_values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(attrs->attislocal); slot[slotCount]->tts_values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(attrs->attinhcount); slot[slotCount]->tts_values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(attrs->attcollation); + slot[slotCount]->tts_values[Anum_pg_attribute_attcompression - 1] = CharGetDatum(attrs->attcompression); if (attoptions && attoptions[natts] != (Datum) 0) slot[slotCount]->tts_values[Anum_pg_attribute_attoptions - 1] = attoptions[natts]; else @@ -1715,6 +1717,8 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) /* Unset this so no one tries to look up the generation expression */ attStruct->attgenerated = '\0'; + attStruct->attcompression = InvalidCompressionMethod; + /* * Change the column name to something that isn't likely to conflict */ diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 4ef61b5efd..397d70d226 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -348,6 +348,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->attbyval = from->attbyval; to->attstorage = from->attstorage; to->attalign = from->attalign; + to->attcompression = from->attcompression; } else { diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index d7b806020d..2196b44aae 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -14,6 +14,7 @@ */ #include "postgres.h" +#include "access/toast_compression.h" #include "access/heapam.h" #include "access/xact.h" #include "catalog/binary_upgrade.h" @@ -220,6 +221,11 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, TupleDescAttr(tupdesc, 1)->attstorage = TYPSTORAGE_PLAIN; TupleDescAttr(tupdesc, 2)->attstorage = TYPSTORAGE_PLAIN; + /* Toast field should not be compressed */ + TupleDescAttr(tupdesc, 0)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 1)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 2)->attcompression = InvalidCompressionMethod; + /* * Toast tables for regular relations go in pg_toast; those for temp * relations go into the per-backend temp-toast-table namespace. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 559fa1d2e5..d295d85cd2 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/attmap.h" +#include "access/toast_compression.h" #include "access/genam.h" #include "access/heapam.h" #include "access/heapam_xlog.h" @@ -558,6 +559,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx, static List *GetParentedForeignKeyRefs(Relation partition); static void ATDetachCheckNoForeignKeyRefs(Relation partition); static void ATExecAlterCollationRefreshVersion(Relation rel, List *coll); +static char GetAttributeCompression(Form_pg_attribute att, char *compression); /* ---------------------------------------------------------------- @@ -852,6 +854,18 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (colDef->generated) attr->attgenerated = colDef->generated; + + /* + * lookup attribute's compression method and store it in the + * attr->attcompression. + */ + if (relkind == RELKIND_RELATION || + relkind == RELKIND_PARTITIONED_TABLE || + relkind == RELKIND_MATVIEW) + attr->attcompression = + GetAttributeCompression(attr, colDef->compression); + else + attr->attcompression = InvalidCompressionMethod; } /* @@ -2396,6 +2410,22 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(attribute->attstorage)))); + /* Copy/check compression parameter */ + if (CompressionMethodIsValid(attribute->attcompression)) + { + const char *compression = GetCompressionMethodName( + attribute->attcompression); + + if (def->compression == NULL) + def->compression = pstrdup(compression); + else if (strcmp(def->compression, compression) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression method conflict", + attributeName), + errdetail("%s versus %s", def->compression, compression))); + } + def->inhcount++; /* Merge of NOT NULL constraints = OR 'em together */ def->is_not_null |= attribute->attnotnull; @@ -2430,6 +2460,11 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collOid = attribute->attcollation; def->constraints = NIL; def->location = -1; + if (CompressionMethodIsValid(attribute->attcompression)) + def->compression = pstrdup(GetCompressionMethodName( + attribute->attcompression)); + else + def->compression = NULL; inhSchema = lappend(inhSchema, def); newattmap->attnums[parent_attno - 1] = ++child_attno; } @@ -2675,6 +2710,19 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(newdef->storage)))); + /* Copy compression parameter */ + if (def->compression == NULL) + def->compression = newdef->compression; + else if (newdef->compression != NULL) + { + if (strcmp(def->compression, newdef->compression) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression method conflict", + attributeName), + errdetail("%s versus %s", def->compression, newdef->compression))); + } + /* Mark the column as locally defined */ def->is_local = true; /* Merge of NOT NULL constraints = OR 'em together */ @@ -6340,6 +6388,18 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.attislocal = colDef->is_local; attribute.attinhcount = colDef->inhcount; attribute.attcollation = collOid; + + /* + * lookup attribute's compression method and store it in the + * attr->attcompression. + */ + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + attribute.attcompression = GetAttributeCompression(&attribute, + colDef->compression); + else + attribute.attcompression = InvalidCompressionMethod; + /* attribute.attacl is handled by InsertPgAttributeTuples() */ ReleaseSysCache(typeTuple); @@ -11859,6 +11919,23 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, ReleaseSysCache(typeTuple); + /* Setup attribute compression */ + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* + * No compression for plain/external storage, otherwise, default + * compression method if it is not already set, refer comments atop + * attcompression parameter in pg_attribute.h. + */ + if (!IsStorageCompressible(tform->typstorage)) + attTup->attcompression = InvalidCompressionMethod; + else if (!CompressionMethodIsValid(attTup->attcompression)) + attTup->attcompression = DefaultCompressionMethod; + } + else + attTup->attcompression = InvalidCompressionMethod; + CatalogTupleUpdate(attrelation, &heapTup->t_self, heapTup); table_close(attrelation, RowExclusiveLock); @@ -17641,3 +17718,36 @@ ATExecAlterCollationRefreshVersion(Relation rel, List *coll) index_update_collation_versions(rel->rd_id, get_collation_oid(coll, false)); CacheInvalidateRelcache(rel); } + +/* + * resolve column compression specification to compression method. + */ +static char +GetAttributeCompression(Form_pg_attribute att, char *compression) +{ + char typstorage = get_typstorage(att->atttypid); + char cmethod; + + /* + * No compression for plain/external storage, refer comments atop + * attcompression parameter in pg_attribute.h + */ + if (!IsStorageCompressible(typstorage)) + { + if (compression == NULL) + return InvalidCompressionMethod; + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column data type %s does not support compression", + format_type_be(att->atttypid)))); + } + + /* fallback to default compression if it's not specified */ + if (compression == NULL) + return DefaultCompressionMethod; + + cmethod = CompressionNameToMethod(compression); + + return cmethod; +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index da91cbd2b1..0d3b923a38 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2988,6 +2988,7 @@ _copyColumnDef(const ColumnDef *from) COPY_STRING_FIELD(colname); COPY_NODE_FIELD(typeName); + COPY_STRING_FIELD(compression); COPY_SCALAR_FIELD(inhcount); COPY_SCALAR_FIELD(is_local); COPY_SCALAR_FIELD(is_not_null); diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index c2d73626fc..f3592003da 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2599,6 +2599,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) { COMPARE_STRING_FIELD(colname); COMPARE_NODE_FIELD(typeName); + COMPARE_STRING_FIELD(compression); COMPARE_SCALAR_FIELD(inhcount); COMPARE_SCALAR_FIELD(is_local); COMPARE_SCALAR_FIELD(is_not_null); diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 49357ac5c2..38226530c6 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -3897,6 +3897,8 @@ raw_expression_tree_walker(Node *node, if (walker(coldef->typeName, context)) return true; + if (walker(coldef->compression, context)) + return true; if (walker(coldef->raw_default, context)) return true; if (walker(coldef->collClause, context)) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 6493a03ff8..75343b8aaf 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2876,6 +2876,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_STRING_FIELD(colname); WRITE_NODE_FIELD(typeName); + WRITE_STRING_FIELD(compression); WRITE_INT_FIELD(inhcount); WRITE_BOOL_FIELD(is_local); WRITE_BOOL_FIELD(is_not_null); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 652be0b96d..9d923b5d95 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -596,6 +596,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <list> hash_partbound %type <defelt> hash_partbound_elem +%type <str> optColumnCompression + /* * Non-keyword token types. These are hard-wired into the "flex" lexer. * They must be listed first so that their numeric codes do not depend on @@ -631,9 +633,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT - COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT - CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE - CROSS CSV CUBE CURRENT_P + COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT + CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY + COST CREATE CROSS CSV CUBE CURRENT_P CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE @@ -3421,11 +3423,12 @@ TypedTableElement: | TableConstraint { $$ = $1; } ; -columnDef: ColId Typename create_generic_options ColQualList +columnDef: ColId Typename optColumnCompression create_generic_options ColQualList { ColumnDef *n = makeNode(ColumnDef); n->colname = $1; n->typeName = $2; + n->compression = $3; n->inhcount = 0; n->is_local = true; n->is_not_null = false; @@ -3434,8 +3437,8 @@ columnDef: ColId Typename create_generic_options ColQualList n->raw_default = NULL; n->cooked_default = NULL; n->collOid = InvalidOid; - n->fdwoptions = $3; - SplitColQualList($4, &n->constraints, &n->collClause, + n->fdwoptions = $4; + SplitColQualList($5, &n->constraints, &n->collClause, yyscanner); n->location = @1; $$ = (Node *)n; @@ -3480,6 +3483,14 @@ columnOptions: ColId ColQualList } ; +optColumnCompression: + COMPRESSION name + { + $$ = $2; + } + | /*EMPTY*/ { $$ = NULL; } + ; + ColQualList: ColQualList ColConstraint { $$ = lappend($1, $2); } | /*EMPTY*/ { $$ = NIL; } @@ -3710,6 +3721,7 @@ TableLikeOption: | INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; } | STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; } | STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; } + | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; } | ALL { $$ = CREATE_TABLE_LIKE_ALL; } ; @@ -15296,6 +15308,7 @@ unreserved_keyword: | COMMENTS | COMMIT | COMMITTED + | COMPRESSION | CONFIGURATION | CONFLICT | CONNECTION @@ -15816,6 +15829,7 @@ bare_label_keyword: | COMMENTS | COMMIT | COMMITTED + | COMPRESSION | CONCURRENTLY | CONFIGURATION | CONFLICT diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 75266caeb4..af772ba6bc 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -27,6 +27,7 @@ #include "postgres.h" #include "access/amapi.h" +#include "access/toast_compression.h" #include "access/htup_details.h" #include "access/relation.h" #include "access/reloptions.h" @@ -1082,6 +1083,14 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla else def->storage = 0; + /* Likewise, copy compression if requested */ + if ((table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION) != 0 + && CompressionMethodIsValid(attribute->attcompression)) + def->compression = + pstrdup(GetCompressionMethodName(attribute->attcompression)); + else + def->compression = NULL; + /* Likewise, copy comment if requested */ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) && (comment = GetComment(attribute->attrelid, diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 479ed9ae54..37876a8cb7 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -17,6 +17,7 @@ #include <ctype.h> #include <limits.h> +#include "access/toast_compression.h" #include "access/detoast.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" @@ -5299,6 +5300,46 @@ pg_column_size(PG_FUNCTION_ARGS) PG_RETURN_INT32(result); } +/* + * Return the compression method stored in the compressed attribute. Return + * NULL for non varlena type or uncompressed data. + */ +Datum +pg_column_compression(PG_FUNCTION_ARGS) +{ + Datum value = PG_GETARG_DATUM(0); + int typlen; + char method; + + /* On first call, get the input type's typlen, and save at *fn_extra */ + if (fcinfo->flinfo->fn_extra == NULL) + { + /* Lookup the datatype of the supplied argument */ + Oid argtypeid = get_fn_expr_argtype(fcinfo->flinfo, 0); + + typlen = get_typlen(argtypeid); + if (typlen == 0) /* should not happen */ + elog(ERROR, "cache lookup failed for type %u", argtypeid); + + fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, + sizeof(int)); + *((int *) fcinfo->flinfo->fn_extra) = typlen; + } + else + typlen = *((int *) fcinfo->flinfo->fn_extra); + + if (typlen != -1) + PG_RETURN_NULL(); + + method = toast_get_compression_method( + (struct varlena *) DatumGetPointer(value)); + + if (!CompressionMethodIsValid(method)) + PG_RETURN_NULL(); + else + PG_RETURN_TEXT_P(cstring_to_text(GetCompressionMethodName(method))); +} + /* * string_agg - Concatenates values and returns string. * diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index eea9f30a79..0296b9bb5e 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -160,6 +160,7 @@ typedef struct _dumpOptions int no_subscriptions; int no_synchronized_snapshots; int no_unlogged_table_data; + int no_toast_compression; int serializable_deferrable; int disable_triggers; int outputNoTablespaces; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index eb988d7eb4..cef9bbd76a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -387,6 +387,7 @@ main(int argc, char **argv) {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, + {"no-toast-compression", no_argument, &dopt.no_toast_compression, 1}, {"no-sync", no_argument, NULL, 7}, {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 10}, @@ -1047,6 +1048,7 @@ help(const char *progname) printf(_(" --no-publications do not dump publications\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); + printf(_(" --no-toast-compression do not dump toast compression methods\n")); printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); @@ -8617,6 +8619,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) { DumpOptions *dopt = fout->dopt; PQExpBuffer q = createPQExpBuffer(); + bool createWithCompression; for (int i = 0; i < numTables; i++) { @@ -8702,6 +8705,15 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) appendPQExpBufferStr(q, "'' AS attidentity,\n"); + createWithCompression = (fout->remoteVersion >= 140000); + + if (createWithCompression) + appendPQExpBuffer(q, + "a.attcompression AS attcompression,\n"); + else + appendPQExpBuffer(q, + "NULL AS attcmname,\n"); + if (fout->remoteVersion >= 110000) appendPQExpBufferStr(q, "CASE WHEN a.atthasmissing AND NOT a.attisdropped " @@ -8747,6 +8759,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid)); tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcompression = (char *) pg_malloc(ntups * sizeof(char *)); tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *)); @@ -8775,6 +8788,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, PQfnumber(res, "attcollation"))); tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attfdwoptions"))); tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attmissingval"))); + tbinfo->attcompression[j] = *(PQgetvalue(res, j, PQfnumber(res, "attcompression"))); tbinfo->attrdefs[j] = NULL; /* fix below */ if (PQgetvalue(res, j, PQfnumber(res, "atthasdef"))[0] == 't') hasdefaults = true; @@ -15891,6 +15905,31 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) tbinfo->atttypnames[j]); } + /* + * Attribute compression + */ + if (!dopt->no_toast_compression && + tbinfo->attcompression[j] && + tbinfo->attcompression[j] != '\0') + { + char *cmname; + + switch (tbinfo->attcompression[j]) + { + case 'p': + cmname = "pglz"; + break; + case 'l': + cmname = "lz4"; + break; + default: + cmname = NULL; + } + + if (cmname != NULL) + appendPQExpBuffer(q, " COMPRESSION %s", cmname); + } + if (print_default) { if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 0a2213fb06..d956b035f3 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -326,6 +326,7 @@ typedef struct _tableInfo char *partbound; /* partition bound definition */ bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */ char *amname; /* relation access method */ + char *attcompression; /* per-attribute current compression method */ /* * Stuff computed only for dumpable tables. diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 737e46464a..bc91bb12ac 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2284,9 +2284,9 @@ my %tests = ( regexp => qr/^ \QCREATE TABLE dump_test.test_table (\E\n \s+\Qcol1 integer NOT NULL,\E\n - \s+\Qcol2 text,\E\n - \s+\Qcol3 text,\E\n - \s+\Qcol4 text,\E\n + \s+\Qcol2 text COMPRESSION\E\D*,\n + \s+\Qcol3 text COMPRESSION\E\D*,\n + \s+\Qcol4 text COMPRESSION\E\D*,\n \s+\QCONSTRAINT test_table_col1_check CHECK ((col1 <= 1000))\E\n \Q)\E\n \QWITH (autovacuum_enabled='false', fillfactor='80');\E\n/xm, @@ -2326,7 +2326,7 @@ my %tests = ( regexp => qr/^ \QCREATE TABLE dump_test.test_second_table (\E \n\s+\Qcol1 integer,\E - \n\s+\Qcol2 text\E + \n\s+\Qcol2 text COMPRESSION\E\D* \n\); /xm, like => @@ -2441,7 +2441,7 @@ my %tests = ( \n\s+\Qcol1 integer,\E \n\s+\Qcol2 boolean,\E \n\s+\Qcol3 boolean,\E - \n\s+\Qcol4 bit(5),\E + \n\s+\Qcol4 bit(5) COMPRESSION\E\D*, \n\s+\Qcol5 double precision\E \n\); /xm, @@ -2459,7 +2459,7 @@ my %tests = ( regexp => qr/^ \QCREATE TABLE dump_test.test_table_identity (\E\n \s+\Qcol1 integer NOT NULL,\E\n - \s+\Qcol2 text\E\n + \s+\Qcol2 text COMPRESSION\E\D*\n \); .* \QALTER TABLE dump_test.test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 20af5a92b4..b284113d55 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1459,7 +1459,7 @@ describeOneTableDetails(const char *schemaname, bool printTableInitialized = false; int i; char *view_def = NULL; - char *headers[11]; + char *headers[12]; PQExpBufferData title; PQExpBufferData tmpbuf; int cols; @@ -1475,7 +1475,8 @@ describeOneTableDetails(const char *schemaname, fdwopts_col = -1, attstorage_col = -1, attstattarget_col = -1, - attdescr_col = -1; + attdescr_col = -1, + attcompression_col = -1; int numrows; struct { @@ -1892,6 +1893,17 @@ describeOneTableDetails(const char *schemaname, appendPQExpBufferStr(&buf, ",\n a.attstorage"); attstorage_col = cols++; + /* compresssion info */ + if (pset.sversion >= 140000 && + !pset.hide_compression && + (tableinfo.relkind == RELKIND_RELATION || + tableinfo.relkind == RELKIND_PARTITIONED_TABLE || + tableinfo.relkind == RELKIND_MATVIEW)) + { + appendPQExpBufferStr(&buf, ",\n a.attcompression AS attcompression"); + attcompression_col = cols++; + } + /* stats target, if relevant to relkind */ if (tableinfo.relkind == RELKIND_RELATION || tableinfo.relkind == RELKIND_INDEX || @@ -2018,6 +2030,8 @@ describeOneTableDetails(const char *schemaname, headers[cols++] = gettext_noop("FDW options"); if (attstorage_col >= 0) headers[cols++] = gettext_noop("Storage"); + if (attcompression_col >= 0) + headers[cols++] = gettext_noop("Compression"); if (attstattarget_col >= 0) headers[cols++] = gettext_noop("Stats target"); if (attdescr_col >= 0) @@ -2097,6 +2111,19 @@ describeOneTableDetails(const char *schemaname, false, false); } + /* Column compression. */ + if (attcompression_col >= 0) + { + char *compression = PQgetvalue(res, i, attcompression_col); + + /* these strings are literal in our syntax, so not translated. */ + printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" : + (compression[0] == 'l' ? "lz4" : + (compression[0] == '\0' ? "" : + "???"))), + false, false); + } + /* Statistics target, if the relkind supports this feature */ if (attstattarget_col >= 0) printTableAddCell(&cont, PQgetvalue(res, i, attstattarget_col), diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index daa5081eac..99a59470c5 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -372,6 +372,8 @@ helpVariables(unsigned short int pager) " true if last query failed, else false\n")); fprintf(output, _(" FETCH_COUNT\n" " the number of result rows to fetch and display at a time (0 = unlimited)\n")); + fprintf(output, _(" HIDE_TOAST_COMPRESSION\n" + " if set, compression methods are not displayed\n")); fprintf(output, _(" HIDE_TABLEAM\n" " if set, table access methods are not displayed\n")); fprintf(output, _(" HISTCONTROL\n" diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index d65990059d..83f2e6f254 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -134,6 +134,7 @@ typedef struct _psqlSettings bool quiet; bool singleline; bool singlestep; + bool hide_compression; bool hide_tableam; int fetch_count; int histsize; diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 780479c8d7..110906a4e9 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -1159,6 +1159,13 @@ show_context_hook(const char *newval) return true; } +static bool +hide_compression_hook(const char *newval) +{ + return ParseVariableBool(newval, "HIDE_TOAST_COMPRESSION", + &pset.hide_compression); +} + static bool hide_tableam_hook(const char *newval) { @@ -1227,6 +1234,9 @@ EstablishVariableSpace(void) SetVariableHooks(pset.vars, "SHOW_CONTEXT", show_context_substitute_hook, show_context_hook); + SetVariableHooks(pset.vars, "HIDE_TOAST_COMPRESSION", + bool_substitute_hook, + hide_compression_hook); SetVariableHooks(pset.vars, "HIDE_TABLEAM", bool_substitute_hook, hide_tableam_hook); diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h index 0adf53c77b..9a428aae2d 100644 --- a/src/include/access/detoast.h +++ b/src/include/access/detoast.h @@ -89,4 +89,12 @@ extern Size toast_raw_datum_size(Datum value); */ extern Size toast_datum_size(Datum value); +/* ---------- + * toast_get_compression_method - + * + * Return the compression method from the compressed value + * ---------- + */ +extern char toast_get_compression_method(struct varlena *attr); + #endif /* DETOAST_H */ diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h new file mode 100644 index 0000000000..38800cb97a --- /dev/null +++ b/src/include/access/toast_compression.h @@ -0,0 +1,119 @@ +/*------------------------------------------------------------------------- + * + * toast_compression.h + * Functions for toast compression. + * + * Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/include/access/toast_compression.h + * + *------------------------------------------------------------------------- + */ + +#ifndef TOAST_COMPRESSION_H +#define TOAST_COMPRESSION_H + +#include "postgres.h" + +/* + * Built-in compression methods. pg_attribute will store this in the + * attcompression column. + */ +#define PGLZ_COMPRESSION 'p' +#define LZ4_COMPRESSION 'l' + +#define InvalidCompressionMethod '\0' +#define CompressionMethodIsValid(cm) ((bool) ((cm) != InvalidCompressionMethod)) + +/* + * Built-in compression method-id. The toast compression header will store + * this in the first 2 bits of the raw length. These built-in compression + * method-id are directly mapped to the built-in compression methods. + */ +typedef enum CompressionId +{ + PGLZ_COMPRESSION_ID = 0, + LZ4_COMPRESSION_ID = 1 +} CompressionId; + +/* use default compression method if it is not specified. */ +#define DefaultCompressionMethod PGLZ_COMPRESSION +#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod) + +#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \ + (storage) != TYPSTORAGE_EXTERNAL) + +/* compression handler routines */ +typedef struct varlena *(*cmcompress_function) (const struct varlena *value); +typedef struct varlena *(*cmdecompress_function) (const struct varlena *value); +typedef struct varlena *(*cmdecompress_slice_function) + (const struct varlena *value, int32 slicelength); + +/* + * API struct for a compression routines. + * + * 'cmname' - name of the compression method + * 'datum_compress' - varlena compression function. + * 'datum_decompress' - varlena decompression function. + * 'datum_decompress_slice' - varlena slice decompression functions. + */ +typedef struct CompressionRoutine +{ + char cmname[64]; + cmcompress_function datum_compress; + cmdecompress_function datum_decompress; + cmdecompress_slice_function datum_decompress_slice; +} CompressionRoutine; + +extern char CompressionNameToMethod(char *compression); +extern const CompressionRoutine *GetCompressionRoutines(char method); + +/* + * CompressionMethodToId - Convert compression method to compression id. + * + * For more details refer comment atop CompressionId in toast_compression.h + */ +static inline CompressionId +CompressionMethodToId(char method) +{ + switch (method) + { + case PGLZ_COMPRESSION: + return PGLZ_COMPRESSION_ID; + case LZ4_COMPRESSION: + return LZ4_COMPRESSION_ID; + default: + elog(ERROR, "invalid compression method %c", method); + } +} + +/* + * CompressionIdToMethod - Convert compression id to compression method + * + * For more details refer comment atop CompressionId in toast_compression.h + */ +static inline Oid +CompressionIdToMethod(CompressionId cmid) +{ + switch (cmid) + { + case PGLZ_COMPRESSION_ID: + return PGLZ_COMPRESSION; + case LZ4_COMPRESSION_ID: + return LZ4_COMPRESSION; + default: + elog(ERROR, "invalid compression method id %d", cmid); + } +} + +/* + * GetCompressionMethodName - Get compression method name + */ +static inline const char* +GetCompressionMethodName(char method) +{ + return GetCompressionRoutines(method)->cmname; +} + + +#endif /* TOAST_COMPRESSION_H */ diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h index a9a6d644bc..05104ce237 100644 --- a/src/include/access/toast_helper.h +++ b/src/include/access/toast_helper.h @@ -32,6 +32,7 @@ typedef struct struct varlena *tai_oldexternal; int32 tai_size; uint8 tai_colflags; + char tai_compression; } ToastAttrInfo; /* diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index cedfb890d8..284a15761a 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -12,6 +12,7 @@ #ifndef TOAST_INTERNALS_H #define TOAST_INTERNALS_H +#include "access/toast_compression.h" #include "storage/lockdefs.h" #include "utils/relcache.h" #include "utils/snapshot.h" @@ -22,22 +23,23 @@ typedef struct toast_compress_header { int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 rawsize; + uint32 tcinfo; /* 2 bits for compression method and 30 bits + * rawsize */ } toast_compress_header; /* * Utilities for manipulation of header information for compressed * toast entries. */ -#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) -#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize) -#define TOAST_COMPRESS_SIZE(ptr) ((int32) VARSIZE_ANY(ptr) - TOAST_COMPRESS_HDRSZ) -#define TOAST_COMPRESS_RAWDATA(ptr) \ - (((char *) (ptr)) + TOAST_COMPRESS_HDRSZ) -#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \ - (((toast_compress_header *) (ptr))->rawsize = (len)) +#define TOAST_COMPRESS_METHOD(ptr) (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_RAWSIZE_BITS) +#define TOAST_COMPRESS_SET_SIZE_AND_METHOD(ptr, len, cm_method) \ + do { \ + Assert((len) > 0 && (len) <= VARLENA_RAWSIZE_MASK); \ + Assert((cm_method) >= PGLZ_COMPRESSION_ID && (cm_method) <= LZ4_COMPRESSION_ID); \ + ((toast_compress_header *) (ptr))->tcinfo = ((len) | (cm_method) << VARLENA_RAWSIZE_BITS); \ + } while (0) -extern Datum toast_compress_datum(Datum value); +extern Datum toast_compress_datum(Datum value, char cmethod); extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index 3db42abf08..560f8f00bb 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -160,6 +160,12 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, /* attribute's collation, if any */ Oid attcollation BKI_LOOKUP_OPT(pg_collation); + /* + * compression method. Must be InvalidCompressionMethod if and only if + * typstorage is 'plain' or 'external'. + */ + char attcompression BKI_DEFAULT('\0'); + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* NOTE: The following fields are not present in tuple descriptors. */ @@ -187,7 +193,7 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, * can access fields beyond attcollation except in a real tuple! */ #define ATTRIBUTE_FIXED_PART_SIZE \ - (offsetof(FormData_pg_attribute,attcollation) + sizeof(Oid)) + (offsetof(FormData_pg_attribute,attcompression) + sizeof(char)) /* ---------------- * Form_pg_attribute corresponds to a pointer to a tuple with diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index c7619f8cd3..91c1c5b720 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -7099,6 +7099,10 @@ descr => 'bytes required to store the value, perhaps with compression', proname => 'pg_column_size', provolatile => 's', prorettype => 'int4', proargtypes => 'any', prosrc => 'pg_column_size' }, +{ oid => '2121', + descr => 'compression method for the compressed datum', + proname => 'pg_column_compression', provolatile => 's', prorettype => 'text', + proargtypes => 'any', prosrc => 'pg_column_compression' }, { oid => '2322', descr => 'total disk space usage for the specified tablespace', proname => 'pg_tablespace_size', provolatile => 'v', prorettype => 'int8', diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 236832a2ca..19d2ba26bf 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -646,6 +646,7 @@ typedef struct ColumnDef NodeTag type; char *colname; /* name of column */ TypeName *typeName; /* type of column */ + char *compression; /* compression method for column */ int inhcount; /* number of times column is inherited */ bool is_local; /* column has local (non-inherited) def'n */ bool is_not_null; /* NOT NULL constraint specified? */ @@ -685,6 +686,7 @@ typedef enum TableLikeOption CREATE_TABLE_LIKE_INDEXES = 1 << 5, CREATE_TABLE_LIKE_STATISTICS = 1 << 6, CREATE_TABLE_LIKE_STORAGE = 1 << 7, + CREATE_TABLE_LIKE_COMPRESSION = 1 << 8, CREATE_TABLE_LIKE_ALL = PG_INT32_MAX } TableLikeOption; diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 28083aaac9..ca1f950cbe 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..6007d72a73 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -346,6 +346,9 @@ /* Define to 1 if you have the `z' library (-lz). */ #undef HAVE_LIBZ +/* Define to 1 if you have the `lz4' library (-llz4). */ +#undef HAVE_LIBLZ4 + /* Define to 1 if you have the `link' function. */ #undef HAVE_LINK diff --git a/src/include/postgres.h b/src/include/postgres.h index 2ed572004d..e98af05b30 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -145,7 +145,8 @@ typedef union struct /* Compressed-in-line format */ { uint32 va_header; - uint32 va_rawsize; /* Original data size (excludes header) */ + uint32 va_tcinfo; /* Original data size (excludes header) and + * compression method */ char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ } va_compressed; } varattrib_4b; @@ -274,14 +275,23 @@ typedef struct (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) #define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data) +#define VARHDRSZ_COMPRESS offsetof(varattrib_4b, va_compressed.va_data) #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) #define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) #define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data) +#define VARLENA_RAWSIZE_BITS 30 +#define VARLENA_RAWSIZE_MASK ((1U << VARLENA_RAWSIZE_BITS) - 1) + +/* + * va_tcinfo in va_compress contains raw size of datum and compression method. + */ #define VARRAWSIZE_4B_C(PTR) \ - (((varattrib_4b *) (PTR))->va_compressed.va_rawsize) + (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_RAWSIZE_MASK) +#define VARCOMPRESS_4B_C(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_RAWSIZE_BITS) /* Externally visible macros */ diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out new file mode 100644 index 0000000000..1de3ae2646 --- /dev/null +++ b/src/test/regress/expected/compression.out @@ -0,0 +1,247 @@ +\set HIDE_TOAST_COMPRESSION false +-- test creating table with compression method +CREATE TABLE cmdata(f1 text COMPRESSION pglz); +CREATE INDEX idx ON cmdata(f1); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +\d+ cmdata + Table "public.cmdata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | +Indexes: + "idx" btree (f1) + +CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); +INSERT INTO cmdata1 VALUES(repeat('1234567890',1004)); +\d+ cmdata1 + Table "public.cmdata1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | lz4 | | + +-- try setting compression for incompressible data type +CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); +ERROR: column data type integer does not support compression +-- verify stored compression method +SELECT pg_column_compression(f1) FROM cmdata; + pg_column_compression +----------------------- + pglz +(1 row) + +SELECT pg_column_compression(f1) FROM cmdata1; + pg_column_compression +----------------------- + lz4 +(1 row) + +-- decompress data slice +SELECT SUBSTR(f1, 200, 5) FROM cmdata; + substr +-------- + 01234 +(1 row) + +SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; + substr +---------------------------------------------------- + 01234567890123456789012345678901234567890123456789 +(1 row) + +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove1; + pg_column_compression +----------------------- + pglz +(1 row) + +-- update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + lz4 +(1 row) + +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz); +INSERT INTO cmmove3 SELECT * FROM cmdata; +INSERT INTO cmmove3 SELECT * FROM cmdata1; +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + lz4 +(1 row) + +-- test external compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); +INSERT INTO cmdata2 select large_val() || repeat('a', 4000); +SELECT pg_column_compression(f1) FROM cmdata2; + pg_column_compression +----------------------- + pglz +(1 row) + +INSERT INTO cmdata1 SELECT * FROM cmdata2; +SELECT pg_column_compression(f1) FROM cmdata1; + pg_column_compression +----------------------- + lz4 + pglz +(2 rows) + +DROP TABLE cmdata2; +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | lz4 | | + +DROP TABLE cmdata2; +--test column type update varlena/non-varlena +CREATE TABLE cmdata2 (f1 int); +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | character varying | | | | extended | pglz | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + +--changing column storage should not impact the compression method +--but the data should not be compressed +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | character varying | | | | extended | pglz | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | character varying | | | | plain | pglz | | + +INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); +SELECT pg_column_compression(f1) FROM cmdata2; + pg_column_compression +----------------------- + +(1 row) + +-- test compression with materialized view +CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; +\d+ mv + Materialized view "public.mv" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + x | text | | | | extended | pglz | | +View definition: + SELECT cmdata1.f1 AS x + FROM cmdata1; + +SELECT pg_column_compression(f1) FROM cmdata1; + pg_column_compression +----------------------- + lz4 + pglz +(2 rows) + +SELECT pg_column_compression(x) FROM mv; + pg_column_compression +----------------------- + lz4 + pglz +(2 rows) + +-- test compression with partition +CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); +CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); +CREATE TABLE cmpart2(f1 text COMPRESSION pglz); +ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +INSERT INTO cmpart VALUES (repeat('123456789',4004)); +SELECT pg_column_compression(f1) FROM cmpart; + pg_column_compression +----------------------- + lz4 + pglz +(2 rows) + +-- test compression with inheritence, error +CREATE TABLE cminh() INHERITS(cmdata, cmdata1); +NOTICE: merging multiple inherited definitions of column "f1" +ERROR: column "f1" has a compression method conflict +DETAIL: pglz versus lz4 +CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); +NOTICE: merging column "f1" with inherited definition +ERROR: column "f1" has a compression method conflict +DETAIL: pglz versus lz4 +-- check data is ok +SELECT length(f1) FROM cmdata; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmdata1; + length +-------- + 10040 + 12449 +(2 rows) + +SELECT length(f1) FROM cmmove1; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10040 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 + 10040 +(2 rows) + +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out new file mode 100644 index 0000000000..2999b3bb79 --- /dev/null +++ b/src/test/regress/expected/compression_1.out @@ -0,0 +1,240 @@ +\set HIDE_TOAST_COMPRESSION false +-- test creating table with compression method +CREATE TABLE cmdata(f1 text COMPRESSION pglz); +CREATE INDEX idx ON cmdata(f1); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +\d+ cmdata + Table "public.cmdata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | +Indexes: + "idx" btree (f1) + +CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); +ERROR: unsupported LZ4 compression method +DETAIL: This functionality requires the server to be built with lz4 support. +HINT: You need to rebuild PostgreSQL using --with-lz4. +INSERT INTO cmdata1 VALUES(repeat('1234567890',1004)); +ERROR: relation "cmdata1" does not exist +LINE 1: INSERT INTO cmdata1 VALUES(repeat('1234567890',1004)); + ^ +\d+ cmdata1 +-- try setting compression for incompressible data type +CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); +ERROR: column data type integer does not support compression +-- verify stored compression method +SELECT pg_column_compression(f1) FROM cmdata; + pg_column_compression +----------------------- + pglz +(1 row) + +SELECT pg_column_compression(f1) FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; + ^ +-- decompress data slice +SELECT SUBSTR(f1, 200, 5) FROM cmdata; + substr +-------- + 01234 +(1 row) + +SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; + ^ +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove1; + pg_column_compression +----------------------- + pglz +(1 row) + +-- update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; + ^ +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz); +INSERT INTO cmmove3 SELECT * FROM cmdata; +INSERT INTO cmmove3 SELECT * FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: INSERT INTO cmmove3 SELECT * FROM cmdata1; + ^ +SELECT pg_column_compression(f1) FROM cmmove2; + pg_column_compression +----------------------- + pglz +(1 row) + +-- test external compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); +INSERT INTO cmdata2 select large_val() || repeat('a', 4000); +SELECT pg_column_compression(f1) FROM cmdata2; + pg_column_compression +----------------------- + pglz +(1 row) + +INSERT INTO cmdata1 SELECT * FROM cmdata2; +ERROR: relation "cmdata1" does not exist +LINE 1: INSERT INTO cmdata1 SELECT * FROM cmdata2; + ^ +SELECT pg_column_compression(f1) FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; + ^ +DROP TABLE cmdata2; +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); +ERROR: relation "cmdata1" does not exist +LINE 1: CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); + ^ +\d+ cmdata2 +DROP TABLE cmdata2; +ERROR: table "cmdata2" does not exist +--test column type update varlena/non-varlena +CREATE TABLE cmdata2 (f1 int); +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | character varying | | | | extended | pglz | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + +--changing column storage should not impact the compression method +--but the data should not be compressed +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | character varying | | | | extended | pglz | | + +ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; +\d+ cmdata2 + Table "public.cmdata2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | character varying | | | | plain | pglz | | + +INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); +SELECT pg_column_compression(f1) FROM cmdata2; + pg_column_compression +----------------------- + +(1 row) + +-- test compression with materialized view +CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; + ^ +\d+ mv +SELECT pg_column_compression(f1) FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmdata1; + ^ +SELECT pg_column_compression(x) FROM mv; +ERROR: relation "mv" does not exist +LINE 1: SELECT pg_column_compression(x) FROM mv; + ^ +-- test compression with partition +CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); +ERROR: unsupported LZ4 compression method +DETAIL: This functionality requires the server to be built with lz4 support. +HINT: You need to rebuild PostgreSQL using --with-lz4. +CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); +ERROR: relation "cmpart" does not exist +CREATE TABLE cmpart2(f1 text COMPRESSION pglz); +ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); +ERROR: relation "cmpart" does not exist +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +ERROR: relation "cmpart" does not exist +LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',1004)); + ^ +INSERT INTO cmpart VALUES (repeat('123456789',4004)); +ERROR: relation "cmpart" does not exist +LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',4004)); + ^ +SELECT pg_column_compression(f1) FROM cmpart; +ERROR: relation "cmpart" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmpart; + ^ +-- test compression with inheritence, error +CREATE TABLE cminh() INHERITS(cmdata, cmdata1); +ERROR: relation "cmdata1" does not exist +CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); +NOTICE: merging column "f1" with inherited definition +ERROR: column "f1" has a compression method conflict +DETAIL: pglz versus lz4 +-- check data is ok +SELECT length(f1) FROM cmdata; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmdata1; +ERROR: relation "cmdata1" does not exist +LINE 1: SELECT length(f1) FROM cmdata1; + ^ +SELECT length(f1) FROM cmmove1; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 +(1 row) + +\set HIDE_TOAST_COMPRESSION true diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e280198b17..70c38309d7 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -115,7 +115,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # ---------- # Another group of parallel tests # ---------- -test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain +test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression # event triggers cannot run concurrently with any test that runs DDL # oidjoins is read-only, though, and should run late for best coverage diff --git a/src/test/regress/pg_regress_main.c b/src/test/regress/pg_regress_main.c index 8dc4941c24..1524676f3b 100644 --- a/src/test/regress/pg_regress_main.c +++ b/src/test/regress/pg_regress_main.c @@ -78,11 +78,11 @@ psql_start_test(const char *testname, * against different AMs without unnecessary differences. */ offset += snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset, - "\"%s%spsql\" -X -a -q -d \"%s\" -v %s < \"%s\" > \"%s\" 2>&1", + "\"%s%spsql\" -X -a -q -d \"%s\" %s < \"%s\" > \"%s\" 2>&1", bindir ? bindir : "", bindir ? "/" : "", dblist->str, - "HIDE_TABLEAM=\"on\"", + "-v HIDE_TABLEAM=on -v HIDE_TOAST_COMPRESSION=on", infile, outfile); if (offset >= sizeof(psql_cmd)) diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 6a57e889a1..d81d04136c 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -201,6 +201,7 @@ test: partition_aggregate test: partition_info test: tuplesort test: explain +test: compression test: event_trigger test: oidjoins test: fast_default diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql new file mode 100644 index 0000000000..ab35f3b4ae --- /dev/null +++ b/src/test/regress/sql/compression.sql @@ -0,0 +1,102 @@ +\set HIDE_TOAST_COMPRESSION false +-- test creating table with compression method +CREATE TABLE cmdata(f1 text COMPRESSION pglz); +CREATE INDEX idx ON cmdata(f1); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +\d+ cmdata + +CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4); +INSERT INTO cmdata1 VALUES(repeat('1234567890',1004)); +\d+ cmdata1 + +-- try setting compression for incompressible data type +CREATE TABLE cmdata2 (f1 int COMPRESSION pglz); + +-- verify stored compression method +SELECT pg_column_compression(f1) FROM cmdata; +SELECT pg_column_compression(f1) FROM cmdata1; + +-- decompress data slice +SELECT SUBSTR(f1, 200, 5) FROM cmdata; +SELECT SUBSTR(f1, 2000, 50) FROM cmdata1; + +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove1; + +-- update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +SELECT pg_column_compression(f1) FROM cmmove2; + +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +SELECT pg_column_compression(f1) FROM cmmove2; +UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1; +SELECT pg_column_compression(f1) FROM cmmove2; + +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz); +INSERT INTO cmmove3 SELECT * FROM cmdata; +INSERT INTO cmmove3 SELECT * FROM cmdata1; +SELECT pg_column_compression(f1) FROM cmmove2; + +-- test external compressed data +CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS +'select array_agg(md5(g::text))::text from generate_series(1, 256) g'; +CREATE TABLE cmdata2 (f1 text COMPRESSION pglz); +INSERT INTO cmdata2 select large_val() || repeat('a', 4000); +SELECT pg_column_compression(f1) FROM cmdata2; +INSERT INTO cmdata1 SELECT * FROM cmdata2; +SELECT pg_column_compression(f1) FROM cmdata1; +DROP TABLE cmdata2; + +-- test LIKE INCLUDING COMPRESSION +CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION); +\d+ cmdata2 +DROP TABLE cmdata2; + +--test column type update varlena/non-varlena +CREATE TABLE cmdata2 (f1 int); +\d+ cmdata2 +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer; +\d+ cmdata2 + +--changing column storage should not impact the compression method +--but the data should not be compressed +ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar; +\d+ cmdata2 +ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain; +\d+ cmdata2 +INSERT INTO cmdata2 VALUES (repeat('123456789', 800)); +SELECT pg_column_compression(f1) FROM cmdata2; + +-- test compression with materialized view +CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1; +\d+ mv +SELECT pg_column_compression(f1) FROM cmdata1; +SELECT pg_column_compression(x) FROM mv; + +-- test compression with partition +CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1); +CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0); +CREATE TABLE cmpart2(f1 text COMPRESSION pglz); + +ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +INSERT INTO cmpart VALUES (repeat('123456789',4004)); +SELECT pg_column_compression(f1) FROM cmpart; + +-- test compression with inheritence, error +CREATE TABLE cminh() INHERITS(cmdata, cmdata1); +CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata); + +-- check data is ok +SELECT length(f1) FROM cmdata; +SELECT length(f1) FROM cmdata1; +SELECT length(f1) FROM cmmove1; +SELECT length(f1) FROM cmmove2; +SELECT length(f1) FROM cmmove3; + +\set HIDE_TOAST_COMPRESSION true diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..5f39a92111 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -307,6 +307,7 @@ sub GenerateFiles HAVE_LIBXML2 => undef, HAVE_LIBXSLT => undef, HAVE_LIBZ => $self->{options}->{zlib} ? 1 : undef, + HAVE_LIBLZ4 => undef, HAVE_LINK => undef, HAVE_LOCALE_T => 1, HAVE_LONG_INT_64 => undef, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 574a8a94fa..e6cb35a16f 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -395,6 +395,7 @@ CompositeIOData CompositeTypeStmt CompoundAffixFlag CompressionAlgorithm +CompressionRoutine CompressorState ComputeXidHorizonsResult ConditionVariable -- 2.17.0 --nHwqXXcoX0o6fKCv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-f-use-pkgconfig.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/4] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) Note that warnings shouldn't be issued during startup (only fatals): $ ./tmp_install/usr/local/pgsql/bin/postgres -D ./testrun/regress/regress/tmp_check/data -c shared_preload_libraries=ab 2023-07-06 14:47:03.817 CDT postmaster[2608106] FATAL: could not access file "ab": No existe el archivo o el directorio 2023-07-06 14:47:03.817 CDT postmaster[2608106] CONTEXT: while loading shared libraries for setting "shared_preload_libraries" --- src/backend/commands/variable.c | 101 ++++++++++++++++++ src/backend/utils/fmgr/dfmgr.c | 3 +- src/backend/utils/misc/guc.c | 6 ++ src/backend/utils/misc/guc_tables.c | 6 +- src/include/fmgr.h | 1 + src/include/utils/guc_hooks.h | 7 ++ .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ 8 files changed, 151 insertions(+), 5 deletions(-) diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c index c9421b17952..0714b4421dc 100644 --- a/src/backend/commands/variable.c +++ b/src/backend/commands/variable.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "utils/tzparser.h" #include "utils/varlena.h" +#include <unistd.h> /* * DATESTYLE @@ -1234,3 +1235,103 @@ check_ssl(bool *newval, void **extra, GucSource source) #endif return true; } + + +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * Issue warnings only during an interactive SET from ALTER + * ROLE/DATABASE/SYSTEM, and not while applying GUCs during startup, + * when relevant errors would be FATAL, and the warnings would be redundant. + * + * IsUnderPostmaster is a hack to allow warning under ALTER SYSTEM but + * not during startup. Both of which use PGC_S_FILE. + */ + if (!IsUnderPostmaster) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + filename = expand_dynamic_library_name(filename); + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages + * from dlopen() are not portable, so it'd be hard to report any + * problem other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server might fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else if (source == PGC_S_TEST) + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index 092004dcf3b..b03017f6275 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -79,7 +79,6 @@ char *Dynamic_library_path; static void *internal_load_library(const char *libname); static void incompatible_module_error(const char *libname, const Pg_magic_struct *module_magic_data) pg_attribute_noreturn(); -static char *expand_dynamic_library_name(const char *name); static void check_restricted_library_name(const char *name); static char *substitute_libpath_macro(const char *name); static char *find_in_dynamic_libpath(const char *basename); @@ -410,7 +409,7 @@ incompatible_module_error(const char *libname, * * The result will always be freshly palloc'd. */ -static char * +char * expand_dynamic_library_name(const char *name) { bool have_slash; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index e2734475189..4cd40ca9678 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4654,6 +4654,12 @@ AlterSystemSetConfigFile(AlterSystemStmt *altersysstmt) union config_var_val newval; void *newextra = NULL; + /* + * PGC_S_FILE is used even though we should use PGC_S_TEST, + * since various GUC validation functions use TEST to give a + * NOTICE rather than an ERROR, and we don't want to allow + * setting values that would always return an error later. + */ if (!parse_and_validate_value(record, name, value, PGC_S_FILE, ERROR, &newval, &newextra)) diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 630ed0f1629..6c424be4c47 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4237,7 +4237,7 @@ struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4248,7 +4248,7 @@ struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4259,7 +4259,7 @@ struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { diff --git a/src/include/fmgr.h b/src/include/fmgr.h index ccb4070a251..c6319782a99 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -749,6 +749,7 @@ extern void **find_rendezvous_variable(const char *varName); extern Size EstimateLibraryStateSpace(void); extern void SerializeLibraryState(Size maxsize, char *start_address); extern void RestoreLibraryState(char *start_address); +char *expand_dynamic_library_name(const char *name); /* * Support for aggregate functions diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 153c652c93e..a11c5488041 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -172,4 +172,11 @@ extern bool check_synchronized_standby_slots(char **newval, void **extra, GucSource source); extern void assign_synchronized_standby_slots(const char *newval, void *extra); +extern bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); + #endif /* GUC_HOOKS_H */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 61396b2a805..de707a4803e 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1083,6 +1083,25 @@ RESET SESSION AUTHORIZATION; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will fail to connect with the new setting. +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will fail to connect with the new setting. +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will fail to connect with the new setting. +DROP DATABASE regression_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index adac36536db..56fb52d4e08 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -494,6 +494,19 @@ RESET SESSION AUTHORIZATION; ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regression_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c -- 2.42.0 --e3dHrZwCppoiYgqG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-errcontext-if-server-fails-to-start-due-to-library-G.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 1/3] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) Note that warnings shouldn't be issued during startup (only fatals): $ ./tmp_install/usr/local/pgsql/bin/postgres -D ./testrun/regress/regress/tmp_check/data -c shared_preload_libraries=ab 2023-07-06 14:47:03.817 CDT postmaster[2608106] FATAL: could not access file "ab": No existe el archivo o el directorio 2023-07-06 14:47:03.817 CDT postmaster[2608106] CONTEXT: while loading shared libraries for setting "shared_preload_libraries" --- src/backend/commands/variable.c | 95 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 6 +- src/include/utils/guc_hooks.h | 7 ++ .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ src/test/regress/expected/rules.out | 8 ++ 6 files changed, 145 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c index 9345131711e..bab51c4572a 100644 --- a/src/backend/commands/variable.c +++ b/src/backend/commands/variable.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "utils/tzparser.h" #include "utils/varlena.h" +#include <unistd.h> /* * DATESTYLE @@ -1234,3 +1235,97 @@ check_ssl(bool *newval, void **extra, GucSource source) #endif return true; } + + +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * issue warnings only during an interactive SET, from ALTER + * ROLE/DATABASE/SYSTEM. + */ + if (source != PGC_S_TEST) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages + * from dlopen() are not portable, so it'd be hard to report any + * problem other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server will currently fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will currently fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 46c258be282..82c662089b6 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4235,7 +4235,7 @@ struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4246,7 +4246,7 @@ struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4257,7 +4257,7 @@ struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index d64dc5fcdb0..78e529ea365 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -178,4 +178,11 @@ extern bool check_standby_slot_names(char **newval, void **extra, GucSource source); extern void assign_standby_slot_names(const char *newval, void *extra); +extern bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); + #endif /* GUC_HOOKS_H */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 61396b2a805..2dcd7cf9ec0 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1083,6 +1083,25 @@ RESET SESSION AUTHORIZATION; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +DROP DATABASE regression_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index adac36536db..56fb52d4e08 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -494,6 +494,19 @@ RESET SESSION AUTHORIZATION; ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regression_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index ef658ad7405..2c494080c54 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3489,6 +3489,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "c:/'a"/path" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +DETAIL: New sessions will currently fail to connect with the new setting. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- 2.42.0 --yJSDWdBuJ1ro2+cx Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-errcontext-if-server-fails-to-start-due-to-library-G.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v8 2/4] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) Note that warnings shouldn't be issued during startup (only fatals): $ ./tmp_install/usr/local/pgsql/bin/postgres -D ./testrun/regress/regress/tmp_check/data -c shared_preload_libraries=ab 2023-07-06 14:47:03.817 CDT postmaster[2608106] FATAL: could not access file "ab": No existe el archivo o el directorio 2023-07-06 14:47:03.817 CDT postmaster[2608106] CONTEXT: while loading shared libraries for setting "shared_preload_libraries" --- src/backend/commands/variable.c | 95 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 6 +- src/include/utils/guc_hooks.h | 7 ++ .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ src/test/regress/expected/rules.out | 8 ++ 6 files changed, 145 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c index 30efcd554ae..7109091440c 100644 --- a/src/backend/commands/variable.c +++ b/src/backend/commands/variable.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "utils/tzparser.h" #include "utils/varlena.h" +#include <unistd.h> /* * DATESTYLE @@ -1234,3 +1235,97 @@ check_ssl(bool *newval, void **extra, GucSource source) #endif return true; } + + +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * issue warnings only during an interactive SET, from ALTER + * ROLE/DATABASE/SYSTEM. + */ + if (source != PGC_S_TEST) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages + * from dlopen() are not portable, so it'd be hard to report any + * problem other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server will currently fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will currently fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index f89f148ff3c..2509167b2d4 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4161,7 +4161,7 @@ struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4172,7 +4172,7 @@ struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4183,7 +4183,7 @@ struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 5300c44f3b0..870006a83fe 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -163,4 +163,11 @@ extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); + #endif /* GUC_HOOKS_H */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 291dd9f8d53..28c2d382595 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1099,6 +1099,25 @@ ERROR: permission denied to examine "session_preload_libraries" DETAIL: Only roles with privileges of the "pg_read_all_settings" role may examine this parameter. ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +DROP DATABASE regression_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index cf72aaf13a3..d957fd3dd0e 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -503,6 +503,19 @@ ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regression_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d878a971df9..7ef2fdf768b 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3486,6 +3486,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "c:/'a"/path" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +DETAIL: New sessions will currently fail to connect with the new setting. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- 2.42.0 --E4B85hAaG7CiJkTa Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0003-errcontext-if-server-fails-to-start-due-to-librar.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/4] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) Note that warnings shouldn't be issued during startup (only fatals): $ ./tmp_install/usr/local/pgsql/bin/postgres -D ./testrun/regress/regress/tmp_check/data -c shared_preload_libraries=ab 2023-07-06 14:47:03.817 CDT postmaster[2608106] FATAL: could not access file "ab": No existe el archivo o el directorio 2023-07-06 14:47:03.817 CDT postmaster[2608106] CONTEXT: while loading shared libraries for setting "shared_preload_libraries" --- src/backend/commands/variable.c | 95 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 6 +- src/include/utils/guc_hooks.h | 7 ++ .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ .../worker_spi/expected/worker_spi.out | 2 + .../modules/worker_spi/sql/worker_spi.sql | 2 + src/test/regress/expected/rules.out | 8 ++ 8 files changed, 149 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c index f0f2e076552..9272a0f0e5a 100644 --- a/src/backend/commands/variable.c +++ b/src/backend/commands/variable.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "utils/tzparser.h" #include "utils/varlena.h" +#include <unistd.h> /* * DATESTYLE @@ -1210,3 +1211,97 @@ check_ssl(bool *newval, void **extra, GucSource source) #endif return true; } + + +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * issue warnings only during an interactive SET, from ALTER + * ROLE/DATABASE/SYSTEM. + */ + if (source != PGC_S_TEST) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages + * from dlopen() are not portable, so it'd be hard to report any + * problem other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server will currently fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will currently fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 59ab630ae40..e4a80d87928 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4101,7 +4101,7 @@ struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4112,7 +4112,7 @@ struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4123,7 +4123,7 @@ struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 2ecb9fc0866..de030d750f1 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -160,4 +160,11 @@ extern void assign_xlog_sync_method(int new_sync_method, void *extra); extern bool check_io_direct(char **newval, void **extra, GucSource source); extern void assign_io_direct(const char *newval, void *extra); +extern bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); + #endif /* GUC_HOOKS_H */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 61396b2a805..2dcd7cf9ec0 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1083,6 +1083,25 @@ RESET SESSION AUTHORIZATION; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +DROP DATABASE regression_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index adac36536db..56fb52d4e08 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -494,6 +494,19 @@ RESET SESSION AUTHORIZATION; ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regression_nosuch_db; +ALTER DATABASE regression_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regression_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regression_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c diff --git a/src/test/modules/worker_spi/expected/worker_spi.out b/src/test/modules/worker_spi/expected/worker_spi.out index dc0a79bf759..5b275669bcf 100644 --- a/src/test/modules/worker_spi/expected/worker_spi.out +++ b/src/test/modules/worker_spi/expected/worker_spi.out @@ -28,6 +28,8 @@ SELECT pg_reload_conf(); t (1 row) +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/modules/worker_spi/sql/worker_spi.sql b/src/test/modules/worker_spi/sql/worker_spi.sql index 4683523b29d..4ed5370d456 100644 --- a/src/test/modules/worker_spi/sql/worker_spi.sql +++ b/src/test/modules/worker_spi/sql/worker_spi.sql @@ -18,6 +18,8 @@ END $$; INSERT INTO schema4.counted VALUES ('total', 0), ('delta', 1); SELECT pg_reload_conf(); +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 7fd81e6a7d0..299c9cb4d2d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3453,6 +3453,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "c:/'a"/path" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +DETAIL: New sessions will currently fail to connect with the new setting. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- 2.34.1 --Cz2/h24zxIeGl+Za Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-errcontext-if-server-fails-to-start-due-to-library-G.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v5 1/3] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) --- src/backend/utils/misc/guc.c | 106 +++++++++++++++++- .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ .../worker_spi/expected/worker_spi.out | 2 + .../modules/worker_spi/sql/worker_spi.sql | 2 + src/test/regress/expected/rules.out | 8 ++ 6 files changed, 147 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index af4a1c30689..4e38e73a277 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -176,6 +176,12 @@ static bool call_enum_check_hook(struct config_enum *conf, int *newval, void **extra, GucSource source, int elevel); static bool check_log_destination(char **newval, void **extra, GucSource source); +static bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); static void assign_log_destination(const char *newval, void *extra); static bool check_wal_consistency_checking(char **newval, void **extra, @@ -4272,7 +4278,7 @@ static struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4283,7 +4289,7 @@ static struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4294,7 +4300,7 @@ static struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { @@ -12506,6 +12512,100 @@ check_max_worker_processes(int *newval, void **extra, GucSource source) return true; } +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * issue warnings only during an interactive SET, from ALTER + * ROLE/DATABASE/SYSTEM. + */ + if (source != PGC_S_TEST && source != PGC_S_FILE) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages from + * dlopen() are not portable, so it'd be hard to report any problem + * other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server will currently fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will currently fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +static bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +static bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +static bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + static bool check_effective_io_concurrency(int *newval, void **extra, GucSource source) { diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 88b1ff843be..ac161ce7c33 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1082,6 +1082,25 @@ RESET SESSION AUTHORIZATION; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +CREATE DATABASE regress_nosuch_db; +ALTER DATABASE regress_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER DATABASE regress_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +DROP DATABASE regress_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index adac36536db..cf605b08d30 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -494,6 +494,19 @@ RESET SESSION AUTHORIZATION; ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regress_nosuch_db; +ALTER DATABASE regress_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regress_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regress_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c diff --git a/src/test/modules/worker_spi/expected/worker_spi.out b/src/test/modules/worker_spi/expected/worker_spi.out index dc0a79bf759..5b275669bcf 100644 --- a/src/test/modules/worker_spi/expected/worker_spi.out +++ b/src/test/modules/worker_spi/expected/worker_spi.out @@ -28,6 +28,8 @@ SELECT pg_reload_conf(); t (1 row) +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/modules/worker_spi/sql/worker_spi.sql b/src/test/modules/worker_spi/sql/worker_spi.sql index 4683523b29d..4ed5370d456 100644 --- a/src/test/modules/worker_spi/sql/worker_spi.sql +++ b/src/test/modules/worker_spi/sql/worker_spi.sql @@ -18,6 +18,8 @@ END $$; INSERT INTO schema4.counted VALUES ('total', 0), ('delta', 1); SELECT pg_reload_conf(); +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 7ec3d2688f0..ea4f9eb39e2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3392,6 +3392,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "c:/'a"/path" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +DETAIL: New sessions will currently fail to connect with the new setting. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- 2.17.1 --3siQDZowHQqNOShm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-errcontext-if-server-fails-to-start-due-to-librar.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v6 2/4] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) TODO: test that warnings aren't issued during startup (only fatals) --- src/backend/commands/variable.c | 96 +++++++++++++++++++ src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/guc_tables.c | 6 +- src/include/utils/guc_hooks.h | 7 ++ .../unsafe_tests/expected/rolenames.out | 19 ++++ .../modules/unsafe_tests/sql/rolenames.sql | 13 +++ .../worker_spi/expected/worker_spi.out | 2 + .../modules/worker_spi/sql/worker_spi.sql | 2 + src/test/regress/expected/rules.out | 8 ++ 9 files changed, 151 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c index 791bac6715c..85642ab8453 100644 --- a/src/backend/commands/variable.c +++ b/src/backend/commands/variable.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "utils/tzparser.h" #include "utils/varlena.h" +#include <unistd.h> /* * DATESTYLE @@ -1210,3 +1211,98 @@ check_ssl(bool *newval, void **extra, GucSource source) #endif return true; } + + +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * issue warnings only during an interactive SET, from ALTER + * ROLE/DATABASE/SYSTEM. + */ + if (source != PGC_S_TEST) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages from + * dlopen() are not portable, so it'd be hard to report any problem + * other than "does not exist". + */ + if (access(filename, R_OK) == 0) + continue; + + + if (source == PGC_S_FILE) + /* ALTER SYSTEM */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("The server will currently fail to start with this setting."), + errhint("If the server is shut down, it will be necessary to manually edit the %s file to allow it to start again.", + "postgresql.auto.conf")); + else + /* ALTER ROLE/DATABASE */ + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errdetail("New sessions will currently fail to connect with the new setting.")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ae8810591d6..9c284906d6e 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -24,7 +24,7 @@ */ #include "postgres.h" -#include <limits.h> +// #include <limits.h> #include <sys/stat.h> #include <unistd.h> diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 05ab087934c..b7c8d8dcdfb 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4015,7 +4015,7 @@ struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4026,7 +4026,7 @@ struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4037,7 +4037,7 @@ struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index f1a9a183b49..3f51436a6f5 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -155,4 +155,11 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern void assign_xlog_sync_method(int new_sync_method, void *extra); +extern bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +extern bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); + #endif /* GUC_HOOKS_H */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 88b1ff843be..ac161ce7c33 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1082,6 +1082,25 @@ RESET SESSION AUTHORIZATION; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +CREATE DATABASE regress_nosuch_db; +ALTER DATABASE regress_nosuch_db SET session_preload_libraries="DoesNotExist"; +WARNING: could not access file "$libdir/plugins/DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +ALTER DATABASE regress_nosuch_db SET local_preload_libraries="DoesNotExist"; +WARNING: could not access file "DoesNotExist" +DETAIL: New sessions will currently fail to connect with the new setting. +DROP DATABASE regress_nosuch_db; +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; -- clean up \c DROP SCHEMA test_roles_schema; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index adac36536db..cf605b08d30 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -494,6 +494,19 @@ RESET SESSION AUTHORIZATION; ROLLBACK; REVOKE pg_read_all_settings FROM regress_role_haspriv; +-- test some warnings +ALTER ROLE regress_testrol0 SET session_preload_libraries="DoesNotExist"; +ALTER ROLE regress_testrol0 SET local_preload_libraries="DoesNotExist"; +CREATE DATABASE regress_nosuch_db; +ALTER DATABASE regress_nosuch_db SET session_preload_libraries="DoesNotExist"; +ALTER DATABASE regress_nosuch_db SET local_preload_libraries="DoesNotExist"; +DROP DATABASE regress_nosuch_db; + +-- SET doesn't do anything, but should not warn, either +SET session_preload_libraries="DoesNotExist"; +SET SESSION session_preload_libraries="DoesNotExist"; +begin; SET LOCAL session_preload_libraries="DoesNotExist"; rollback; + -- clean up \c diff --git a/src/test/modules/worker_spi/expected/worker_spi.out b/src/test/modules/worker_spi/expected/worker_spi.out index dc0a79bf759..5b275669bcf 100644 --- a/src/test/modules/worker_spi/expected/worker_spi.out +++ b/src/test/modules/worker_spi/expected/worker_spi.out @@ -28,6 +28,8 @@ SELECT pg_reload_conf(); t (1 row) +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/modules/worker_spi/sql/worker_spi.sql b/src/test/modules/worker_spi/sql/worker_spi.sql index 4683523b29d..4ed5370d456 100644 --- a/src/test/modules/worker_spi/sql/worker_spi.sql +++ b/src/test/modules/worker_spi/sql/worker_spi.sql @@ -18,6 +18,8 @@ END $$; INSERT INTO schema4.counted VALUES ('total', 0), ('delta', 1); SELECT pg_reload_conf(); +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 624d0e5aae1..3a36898b8d3 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3398,6 +3398,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "c:/'a"/path" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "" +DETAIL: New sessions will currently fail to connect with the new setting. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +DETAIL: New sessions will currently fail to connect with the new setting. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- 2.25.1 --vDrIcvfimaPJgjx+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0003-errcontext-if-server-fails-to-start-due-to-librar.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v4 1/3] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) --- src/backend/utils/misc/guc.c | 103 +++++++++++++++++- .../unsafe_tests/expected/rolenames.out | 1 + .../worker_spi/expected/worker_spi.out | 2 + .../modules/worker_spi/sql/worker_spi.sql | 2 + src/test/regress/expected/rules.out | 9 ++ src/test/regress/sql/rules.sql | 1 + 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index f505413a7f9..c44b8ebbfd6 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -170,6 +170,12 @@ static bool call_enum_check_hook(struct config_enum *conf, int *newval, void **extra, GucSource source, int elevel); static bool check_log_destination(char **newval, void **extra, GucSource source); +static bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); static void assign_log_destination(const char *newval, void *extra); static bool check_wal_consistency_checking(char **newval, void **extra, @@ -4198,7 +4204,7 @@ static struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4209,7 +4215,7 @@ static struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4220,7 +4226,7 @@ static struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { @@ -12149,6 +12155,97 @@ check_max_worker_processes(int *newval, void **extra, GucSource source) return true; } +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted, bool is_session) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* + * Do not issue warnings while applying GUCs during startup. That would + * issue a FATAL error message, and the warnings would be redundant. + */ + if (!IsUnderPostmaster) + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages from + * dlopen() are not portable, so it'd be hard to report any problem + * other than "does not exist". + */ + if (access(filename, R_OK) == 0) // F_OK + continue; + + if (is_session) + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errhint("New sessions will fail with the existing configuration.")); + else + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename), + errhint("The server will fail to start with the existing configuration." + " If it is is shut down, it will be necessary to manually edit the %s file to allow it to start.", + "postgresql.auto.conf")); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +static bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true, false); +} + +static bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false, false); +} + +static bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true, true); +} + static bool check_effective_io_concurrency(int *newval, void **extra, GucSource source) { diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index eb608fdc2ea..368b10558d7 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1066,6 +1066,7 @@ GRANT pg_read_all_settings TO regress_role_haspriv; BEGIN; -- A GUC using GUC_SUPERUSER_ONLY is useful for negative tests. SET LOCAL session_preload_libraries TO 'path-to-preload-libraries'; +WARNING: could not access file "$libdir/plugins/path-to-preload-libraries" SET SESSION AUTHORIZATION regress_role_haspriv; -- passes with role member of pg_read_all_settings SHOW session_preload_libraries; diff --git a/src/test/modules/worker_spi/expected/worker_spi.out b/src/test/modules/worker_spi/expected/worker_spi.out index dc0a79bf759..5b275669bcf 100644 --- a/src/test/modules/worker_spi/expected/worker_spi.out +++ b/src/test/modules/worker_spi/expected/worker_spi.out @@ -28,6 +28,8 @@ SELECT pg_reload_conf(); t (1 row) +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/modules/worker_spi/sql/worker_spi.sql b/src/test/modules/worker_spi/sql/worker_spi.sql index 4683523b29d..4ed5370d456 100644 --- a/src/test/modules/worker_spi/sql/worker_spi.sql +++ b/src/test/modules/worker_spi/sql/worker_spi.sql @@ -18,6 +18,8 @@ END $$; INSERT INTO schema4.counted VALUES ('total', 0), ('delta', 1); SELECT pg_reload_conf(); +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1420288d67b..f3d33dda95e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3362,6 +3362,7 @@ drop table hats; drop table hat_data; -- test for pg_get_functiondef properly regurgitating SET parameters -- Note that the function is kept around to stress pg_dump. +SET check_function_bodies=no; CREATE FUNCTION func_with_set_params() RETURNS integer AS 'select 1;' LANGUAGE SQL @@ -3371,6 +3372,14 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +HINT: The server will fail to start with the existing configuration. If it is is shut down, it will be necessary to manually edit the postgresql.auto.conf file to allow it to start. +WARNING: could not access file "c:/'a"/path" +HINT: The server will fail to start with the existing configuration. If it is is shut down, it will be necessary to manually edit the postgresql.auto.conf file to allow it to start. +WARNING: could not access file "" +HINT: The server will fail to start with the existing configuration. If it is is shut down, it will be necessary to manually edit the postgresql.auto.conf file to allow it to start. +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +HINT: The server will fail to start with the existing configuration. If it is is shut down, it will be necessary to manually edit the postgresql.auto.conf file to allow it to start. SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql index 8bdab6dec30..6308e5d27fd 100644 --- a/src/test/regress/sql/rules.sql +++ b/src/test/regress/sql/rules.sql @@ -1198,6 +1198,7 @@ drop table hat_data; -- test for pg_get_functiondef properly regurgitating SET parameters -- Note that the function is kept around to stress pg_dump. +SET check_function_bodies=no; CREATE FUNCTION func_with_set_params() RETURNS integer AS 'select 1;' LANGUAGE SQL -- 2.17.1 --1Ow488MNN9B9o/ov Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-errcontext-if-server-fails-to-start-due-to-librar.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v3 2/2] warn when setting GUC to a nonextant library @ 2021-12-13 14:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2021-12-13 14:42 UTC (permalink / raw) XXX: the SPI test is unstable: -SELECT * FROM schema4.counted; WARNING: could not load library: $libdir/plugins/worker_spi: cannot open shared object file: No such file or directory +SELECT * FROM schema4.counted; --- src/backend/utils/misc/guc.c | 85 ++++++++++++++++++- .../unsafe_tests/expected/rolenames.out | 1 + .../worker_spi/expected/worker_spi.out | 2 + .../modules/worker_spi/sql/worker_spi.sql | 2 + src/test/regress/expected/rules.out | 5 ++ src/test/regress/sql/rules.sql | 1 + 6 files changed, 93 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 4c94f09c645..5a228a37ee2 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -170,6 +170,12 @@ static bool call_enum_check_hook(struct config_enum *conf, int *newval, void **extra, GucSource source, int elevel); static bool check_log_destination(char **newval, void **extra, GucSource source); +static bool check_local_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_session_preload_libraries(char **newval, void **extra, + GucSource source); +static bool check_shared_preload_libraries(char **newval, void **extra, + GucSource source); static void assign_log_destination(const char *newval, void *extra); static bool check_wal_consistency_checking(char **newval, void **extra, @@ -4188,7 +4194,7 @@ static struct config_string ConfigureNamesString[] = }, &session_preload_libraries_string, "", - NULL, NULL, NULL + check_session_preload_libraries, NULL, NULL }, { @@ -4199,7 +4205,7 @@ static struct config_string ConfigureNamesString[] = }, &shared_preload_libraries_string, "", - NULL, NULL, NULL + check_shared_preload_libraries, NULL, NULL }, { @@ -4210,7 +4216,7 @@ static struct config_string ConfigureNamesString[] = }, &local_preload_libraries_string, "", - NULL, NULL, NULL + check_local_preload_libraries, NULL, NULL }, { @@ -12100,6 +12106,79 @@ check_max_worker_processes(int *newval, void **extra, GucSource source) return true; } +/* + * See also load_libraries() and internal_load_library(). + */ +static bool +check_preload_libraries(char **newval, void **extra, GucSource source, + bool restricted) +{ + char *rawstring; + List *elemlist; + ListCell *l; + + /* nothing to do if empty */ + if (newval == NULL || *newval[0] == '\0') + return true; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + /* Parse string into list of filename paths */ + if (!SplitDirectoriesString(rawstring, ',', &elemlist)) + { + /* Should not happen ? */ + return false; + } + + foreach(l, elemlist) + { + /* Note that filename was already canonicalized */ + char *filename = (char *) lfirst(l); + char *expanded = NULL; + + /* If restricting, insert $libdir/plugins if not mentioned already */ + if (restricted && first_dir_separator(filename) == NULL) + { + expanded = psprintf("$libdir/plugins/%s", filename); + filename = expanded; + } + + /* + * stat()/access() only check that the library exists, not that it has + * the correct magic number or even a library. But error messages from + * dlopen() are not portable, so it'd be hard to report any problem + * other than "does not exist". + */ + if (access(filename, R_OK) == -1) // F_OK + ereport(WARNING, + errcode_for_file_access(), + errmsg("could not access file \"%s\"", filename)); + } + + list_free_deep(elemlist); + pfree(rawstring); + return true; +} + +static bool +check_shared_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + +static bool +check_local_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, false); +} + +static bool +check_session_preload_libraries(char **newval, void **extra, GucSource source) +{ + return check_preload_libraries(newval, extra, source, true); +} + static bool check_effective_io_concurrency(int *newval, void **extra, GucSource source) { diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index eb608fdc2ea..368b10558d7 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1066,6 +1066,7 @@ GRANT pg_read_all_settings TO regress_role_haspriv; BEGIN; -- A GUC using GUC_SUPERUSER_ONLY is useful for negative tests. SET LOCAL session_preload_libraries TO 'path-to-preload-libraries'; +WARNING: could not access file "$libdir/plugins/path-to-preload-libraries" SET SESSION AUTHORIZATION regress_role_haspriv; -- passes with role member of pg_read_all_settings SHOW session_preload_libraries; diff --git a/src/test/modules/worker_spi/expected/worker_spi.out b/src/test/modules/worker_spi/expected/worker_spi.out index dc0a79bf759..5b275669bcf 100644 --- a/src/test/modules/worker_spi/expected/worker_spi.out +++ b/src/test/modules/worker_spi/expected/worker_spi.out @@ -28,6 +28,8 @@ SELECT pg_reload_conf(); t (1 row) +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/modules/worker_spi/sql/worker_spi.sql b/src/test/modules/worker_spi/sql/worker_spi.sql index 4683523b29d..4ed5370d456 100644 --- a/src/test/modules/worker_spi/sql/worker_spi.sql +++ b/src/test/modules/worker_spi/sql/worker_spi.sql @@ -18,6 +18,8 @@ END $$; INSERT INTO schema4.counted VALUES ('total', 0), ('delta', 1); SELECT pg_reload_conf(); +-- reconnect to avoid unstable test result due to asynchronous signal +\c -- wait until the worker has processed the tuple we just inserted DO $$ DECLARE diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d652f7b5fb4..ce396814332 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3574,6 +3574,7 @@ drop table hats; drop table hat_data; -- test for pg_get_functiondef properly regurgitating SET parameters -- Note that the function is kept around to stress pg_dump. +SET check_function_bodies=no; CREATE FUNCTION func_with_set_params() RETURNS integer AS 'select 1;' LANGUAGE SQL @@ -3583,6 +3584,10 @@ CREATE FUNCTION func_with_set_params() RETURNS integer SET datestyle to iso, mdy SET local_preload_libraries TO "Mixed/Case", 'c:/''a"/path', '', '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' IMMUTABLE STRICT; +WARNING: could not access file "Mixed/Case" +WARNING: could not access file "c:/'a"/path" +WARNING: could not access file "" +WARNING: could not access file "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" SELECT pg_get_functiondef('func_with_set_params()'::regprocedure); pg_get_functiondef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql index b732833e63c..1e42d092862 100644 --- a/src/test/regress/sql/rules.sql +++ b/src/test/regress/sql/rules.sql @@ -1198,6 +1198,7 @@ drop table hat_data; -- test for pg_get_functiondef properly regurgitating SET parameters -- Note that the function is kept around to stress pg_dump. +SET check_function_bodies=no; CREATE FUNCTION func_with_set_params() RETURNS integer AS 'select 1;' LANGUAGE SQL -- 2.17.1 --9Q79yvCDNTHf0qBd-- ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2021-12-13 14:42 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-05 04:03 [PATCH 1/5] Built-in compression method Dilip Kumar <[email protected]> 2021-12-13 14:42 [PATCH 1/3] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH v8 2/4] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH 2/4] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH 2/4] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH v5 1/3] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH v6 2/4] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH v4 1/3] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]> 2021-12-13 14:42 [PATCH v3 2/2] warn when setting GUC to a nonextant library Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox