public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 4/8] Built-in compression method
2+ messages / 2 participants
[nested] [flat]
* [PATCH 4/8] Built-in compression method
@ 2021-03-05 04:03 Dilip Kumar <[email protected]>
0 siblings, 0 replies; 2+ messages in thread
From: Dilip Kumar @ 2021-03-05 04:03 UTC (permalink / raw)
Add syntax allowing a compression method to be specified.
As of now there is only 2 option for build-in compression
method (pglz, lz4) which can be set while creating a table
or adding a new column. No option for altering the
compression method for an existing column.
Dilip Kumar based on the patches from Ildus Kurbangaliev.
Design input from Robert Haas and Tomas Vondra.
Reviewed by Robert Haas, Tomas Vondra, Alexander Korotkov and Justin Pryzby
Discussions:
https://www.postgresql.org/message-id/[email protected]
https://www.postgresql.org/message-id/CA%2BTgmoaKDW1Oi9V%3Djc9hOGyf77NbkNEABuqgHD1Cq%3D%3D1QsOcxg%40...
https://www.postgresql.org/message-id/CA%2BTgmobSDVgUage9qQ5P_%3DF_9jaMkCgyKxUQGtFQU7oN4kX-AA%40mail...
https://www.postgresql.org/message-id/20201005160355.byp74sh3ejsv7wrj%40development
https://www.postgresql.org/message-id/CAFiTN-tzTTT2oqWdRGLv1dvvS5MC1W%2BLE%2B3bqWPJUZj4GnHOJg%40mail...
---
configure | 215 ++++++++++++
configure.ac | 19 ++
doc/src/sgml/catalogs.sgml | 10 +
doc/src/sgml/ref/create_table.sgml | 33 +-
doc/src/sgml/ref/psql-ref.sgml | 11 +
src/backend/access/brin/brin_tuple.c | 5 +-
src/backend/access/common/Makefile | 1 +
src/backend/access/common/detoast.c | 71 ++--
src/backend/access/common/indextuple.c | 3 +-
src/backend/access/common/toast_compression.c | 308 ++++++++++++++++++
src/backend/access/common/toast_internals.c | 54 ++-
src/backend/access/common/tupdesc.c | 8 +
src/backend/access/heap/heapam_handler.c | 22 ++
src/backend/access/table/toast_helper.c | 5 +-
src/backend/bootstrap/bootstrap.c | 5 +
src/backend/catalog/genbki.pl | 3 +
src/backend/catalog/heap.c | 4 +
src/backend/catalog/index.c | 1 +
src/backend/catalog/toasting.c | 6 +
src/backend/commands/tablecmds.c | 110 +++++++
src/backend/executor/nodeModifyTable.c | 128 ++++++++
src/backend/nodes/copyfuncs.c | 1 +
src/backend/nodes/equalfuncs.c | 1 +
src/backend/nodes/nodeFuncs.c | 2 +
src/backend/nodes/outfuncs.c | 1 +
src/backend/parser/gram.y | 26 +-
src/backend/parser/parse_utilcmd.c | 9 +
src/backend/utils/adt/varlena.c | 41 +++
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_dump.c | 39 +++
src/bin/pg_dump/pg_dump.h | 1 +
src/bin/pg_dump/t/002_pg_dump.pl | 12 +-
src/bin/psql/describe.c | 31 +-
src/bin/psql/help.c | 2 +
src/bin/psql/settings.h | 1 +
src/bin/psql/startup.c | 10 +
src/include/access/detoast.h | 8 +
src/include/access/toast_compression.h | 119 +++++++
src/include/access/toast_helper.h | 1 +
src/include/access/toast_internals.h | 20 +-
src/include/catalog/pg_attribute.h | 8 +-
src/include/catalog/pg_proc.dat | 4 +
src/include/executor/executor.h | 4 +-
src/include/nodes/execnodes.h | 6 +
src/include/nodes/parsenodes.h | 2 +
src/include/parser/kwlist.h | 1 +
src/include/pg_config.h.in | 3 +
src/include/postgres.h | 14 +-
src/test/regress/expected/compression.out | 267 +++++++++++++++
src/test/regress/expected/compression_1.out | 269 +++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/pg_regress_main.c | 4 +-
src/test/regress/serial_schedule | 1 +
src/test/regress/sql/compression.sql | 113 +++++++
src/tools/msvc/Solution.pm | 1 +
src/tools/pgindent/typedefs.list | 1 +
56 files changed, 1962 insertions(+), 86 deletions(-)
create mode 100644 src/backend/access/common/toast_compression.c
create mode 100644 src/include/access/toast_compression.h
create mode 100644 src/test/regress/expected/compression.out
create mode 100644 src/test/regress/expected/compression_1.out
create mode 100644 src/test/regress/sql/compression.sql
diff --git a/configure b/configure
index fad817bb38..87ed16060e 100755
--- a/configure
+++ b/configure
@@ -654,6 +654,8 @@ UUID_LIBS
LDAP_LIBS_BE
LDAP_LIBS_FE
with_ssl
+LZ4_LIBS
+LZ4_CFLAGS
PTHREAD_CFLAGS
PTHREAD_LIBS
PTHREAD_CC
@@ -699,6 +701,7 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -895,6 +899,8 @@ LDFLAGS_EX
LDFLAGS_SL
PERL
PYTHON
+LZ4_CFLAGS
+LZ4_LIBS
MSGFMT
TCLSH'
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1600,6 +1607,8 @@ Some influential environment variables:
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
PYTHON Python program
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
MSGFMT msgfmt program for NLS
TCLSH Tcl interpreter program (tclsh)
@@ -8563,6 +8572,41 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
#
# Assignments
#
@@ -12110,6 +12154,147 @@ fi
fi
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4_compress in -llz4" >&5
+$as_echo_n "checking for LZ4_compress in -llz4... " >&6; }
+if ${ac_cv_lib_lz4_LZ4_compress+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-llz4 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char LZ4_compress ();
+int
+main ()
+{
+return LZ4_compress ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_lz4_LZ4_compress=yes
+else
+ ac_cv_lib_lz4_LZ4_compress=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lz4_LZ4_compress" >&5
+$as_echo "$ac_cv_lib_lz4_LZ4_compress" >&6; }
+if test "x$ac_cv_lib_lz4_LZ4_compress" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBLZ4 1
+_ACEOF
+
+ LIBS="-llz4 $LIBS"
+
+else
+ as_fn_error $? "library 'lz4' is required for LZ4 support" "$LINENO" 5
+fi
+
+fi
+
if test "$enable_spinlocks" = yes; then
$as_echo "#define HAVE_SPINLOCKS 1" >>confdefs.h
@@ -13376,6 +13561,36 @@ Use --without-zlib to disable zlib support." "$LINENO" 5
fi
+fi
+
+if test "$with_lz4" = yes; then
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
fi
if test "$with_gssapi" = yes ; then
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..c8f199dc5e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,15 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
#
# Assignments
#
@@ -1174,6 +1183,11 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ AC_CHECK_LIB(lz4, LZ4_compress, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])])
+fi
+
if test "$enable_spinlocks" = yes; then
AC_DEFINE(HAVE_SPINLOCKS, 1, [Define to 1 if you have spinlocks.])
else
@@ -1407,6 +1421,11 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index b1de6d0674..5fdc80ff3d 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -1355,6 +1355,16 @@
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>attcompression</structfield> <type>char</type>
+ </para>
+ <para>
+ The current compression method of the column. Must be <literal>InvalidCompressionMethod</literal>
+ if and only if typstorage is 'plain' or 'external'.
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>attacl</structfield> <type>aclitem[]</type>
diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml
index 3b2b227683..7c8fc77983 100644
--- a/doc/src/sgml/ref/create_table.sgml
+++ b/doc/src/sgml/ref/create_table.sgml
@@ -22,7 +22,7 @@ PostgreSQL documentation
<refsynopsisdiv>
<synopsis>
CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] <replaceable class="parameter">table_name</replaceable> ( [
- { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ]
+ { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ COMPRESSION <replaceable>compression_method</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ]
| <replaceable>table_constraint</replaceable>
| LIKE <replaceable>source_table</replaceable> [ <replaceable>like_option</replaceable> ... ] }
[, ... ]
@@ -288,6 +288,26 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>COMPRESSION <replaceable class="parameter">compression_method</replaceable></literal></term>
+ <listitem>
+ <para>
+ The <literal>COMPRESSION</literal> clause sets the compression method
+ for a column. Compression is supported only for variable-width data
+ types, and is used only for columns whose storage type is main or
+ extended. (See <xref linkend="sql-altertable"/> for information on
+ column storage types.) Setting this property for a partitioned table
+ has no direct effect, because such tables have no storage of their own,
+ but the configured value is inherited by newly-created partitions.
+ The supported compression methods are <literal>pglz</literal> and
+ <literal>lz4</literal>. <literal>lz4</literal> is available only if
+ <literal>--with-lz4</literal> was used when building
+ <productname>PostgreSQL</productname>. The default
+ is <literal>pglz</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>INHERITS ( <replaceable>parent_table</replaceable> [, ... ] )</literal></term>
<listitem>
@@ -605,6 +625,17 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>INCLUDING COMPRESSION</literal></term>
+ <listitem>
+ <para>
+ Compression method of the columns will be copied. The default
+ behavior is to exclude compression methods, resulting in columns
+ having the default compression method.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>INCLUDING CONSTRAINTS</literal></term>
<listitem>
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index 13c1edfa4d..01ec9b8b0a 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3863,6 +3863,17 @@ bar
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><varname>HIDE_TOAST_COMPRESSION</varname></term>
+ <listitem>
+ <para>
+ If this variable is set to <literal>true</literal>, column
+ compression method details are not displayed. This is mainly
+ useful for regression tests.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><varname>HIDE_TABLEAM</varname></term>
<listitem>
diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c
index a7eb1c9473..0ab5712c71 100644
--- a/src/backend/access/brin/brin_tuple.c
+++ b/src/backend/access/brin/brin_tuple.c
@@ -213,7 +213,10 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
(atttype->typstorage == TYPSTORAGE_EXTENDED ||
atttype->typstorage == TYPSTORAGE_MAIN))
{
- Datum cvalue = toast_compress_datum(value);
+ Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
+ keyno);
+ Datum cvalue = toast_compress_datum(value,
+ att->attcompression);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile
index 5a007d63f1..b9aff0ccfd 100644
--- a/src/backend/access/common/Makefile
+++ b/src/backend/access/common/Makefile
@@ -25,6 +25,7 @@ OBJS = \
scankey.o \
session.o \
syncscan.o \
+ toast_compression.o \
toast_internals.o \
tupconvert.o \
tupdesc.o
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index d1cdbaf648..9f05a4502b 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -16,6 +16,7 @@
#include "access/detoast.h"
#include "access/table.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/toast_internals.h"
#include "common/int.h"
#include "common/pg_lzcompress.h"
@@ -456,6 +457,37 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
return result;
}
+/* ----------
+ * toast_get_compression_method
+ *
+ * Returns compression method for the compressed varlena. If it is not
+ * compressed then returns InvalidCompressionMethod.
+ */
+char
+toast_get_compression_method(struct varlena *attr)
+{
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+ /* fast path for non-compressed external datums */
+ if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ return InvalidCompressionMethod;
+
+ /*
+ * Fetch just enough of the value to examine the compression header,
+ * so that we can find out the compression method.
+ */
+ attr = toast_fetch_datum_slice(attr, 0, VARHDRSZ);
+ }
+ else if (!VARATT_IS_COMPRESSED(attr))
+ return InvalidCompressionMethod;
+
+ return CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr));
+}
+
/* ----------
* toast_decompress_datum -
*
@@ -464,21 +496,18 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
static struct varlena *
toast_decompress_datum(struct varlena *attr)
{
- struct varlena *result;
+ const CompressionRoutine *cmroutine;
Assert(VARATT_IS_COMPRESSED(attr));
- result = (struct varlena *)
- palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
- SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
-
- if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
- TOAST_COMPRESS_SIZE(attr),
- VARDATA(result),
- TOAST_COMPRESS_RAWSIZE(attr), true) < 0)
- elog(ERROR, "compressed data is corrupted");
+ /*
+ * Get compression handler routines, using the compression id stored in the
+ * toast header.
+ */
+ cmroutine = GetCompressionRoutines(
+ CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr)));
- return result;
+ return cmroutine->datum_decompress(attr);
}
@@ -492,22 +521,18 @@ toast_decompress_datum(struct varlena *attr)
static struct varlena *
toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
{
- struct varlena *result;
- int32 rawsize;
+ const CompressionRoutine *cmroutine;
Assert(VARATT_IS_COMPRESSED(attr));
- result = (struct varlena *) palloc(slicelength + VARHDRSZ);
-
- rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
- VARSIZE(attr) - TOAST_COMPRESS_HDRSZ,
- VARDATA(result),
- slicelength, false);
- if (rawsize < 0)
- elog(ERROR, "compressed data is corrupted");
+ /*
+ * Get compression handler routines, using the compression id stored in the
+ * toast header.
+ */
+ cmroutine = GetCompressionRoutines(
+ CompressionIdToMethod(TOAST_COMPRESS_METHOD(attr)));
- SET_VARSIZE(result, rawsize + VARHDRSZ);
- return result;
+ return cmroutine->datum_decompress_slice(attr, slicelength);
}
/* ----------
diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index b72a138497..1f6b7b77d4 100644
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -103,7 +103,8 @@ index_form_tuple(TupleDesc tupleDescriptor,
(att->attstorage == TYPSTORAGE_EXTENDED ||
att->attstorage == TYPSTORAGE_MAIN))
{
- Datum cvalue = toast_compress_datum(untoasted_values[i]);
+ Datum cvalue = toast_compress_datum(untoasted_values[i],
+ att->attcompression);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
new file mode 100644
index 0000000000..3463b42438
--- /dev/null
+++ b/src/backend/access/common/toast_compression.c
@@ -0,0 +1,308 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_compression.c
+ * Functions for toast compression.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/access/common/toast_compression.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#ifdef HAVE_LIBLZ4
+#include <lz4.h>
+#endif
+
+#include "access/toast_compression.h"
+#include "common/pg_lzcompress.h"
+#include "fmgr.h"
+#include "utils/builtins.h"
+
+static struct varlena *pglz_cmcompress(const struct varlena *value);
+static struct varlena *pglz_cmdecompress(const struct varlena *value);
+static struct varlena *pglz_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength);
+static struct varlena *lz4_cmcompress(const struct varlena *value);
+static struct varlena *lz4_cmdecompress(const struct varlena *value);
+static struct varlena *lz4_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength);
+
+#define NO_LZ4_SUPPORT() \
+ ereport(ERROR, \
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
+ errmsg("unsupported LZ4 compression method"), \
+ errdetail("This functionality requires the server to be built with lz4 support."), \
+ errhint("You need to rebuild PostgreSQL using --with-lz4.")))
+
+/* handler routines for pglz and lz4 built-in compression methods */
+const CompressionRoutine toast_compression[] =
+{
+ {
+ .cmname = "pglz",
+ .datum_compress = pglz_cmcompress,
+ .datum_decompress = pglz_cmdecompress,
+ .datum_decompress_slice = pglz_cmdecompress_slice
+ },
+ {
+ .cmname = "lz4",
+ .datum_compress = lz4_cmcompress,
+ .datum_decompress = lz4_cmdecompress,
+ .datum_decompress_slice = lz4_cmdecompress_slice
+ }
+};
+
+/*
+ * pglz_cmcompress - compression routine for pglz compression method
+ *
+ * Compresses source into dest using the default strategy. Returns the
+ * compressed varlena, or NULL if compression fails.
+ */
+static struct varlena *
+pglz_cmcompress(const struct varlena *value)
+{
+ int32 valsize,
+ len;
+ struct varlena *tmp = NULL;
+
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+
+ /*
+ * No point in wasting a palloc cycle if value size is outside the allowed
+ * range for compression.
+ */
+ if (valsize < PGLZ_strategy_default->min_input_size ||
+ valsize > PGLZ_strategy_default->max_input_size)
+ return NULL;
+
+ /*
+ * Figure out the maximum possible size of the pglz output, add the bytes
+ * that will be needed for varlena overhead, and allocate that amount.
+ */
+ tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
+ VARHDRSZ_COMPRESS);
+
+ len = pglz_compress(VARDATA_ANY(value),
+ valsize,
+ (char *) tmp + VARHDRSZ_COMPRESS,
+ NULL);
+ if (len < 0)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS);
+
+ return tmp;
+}
+
+/*
+ * pglz_cmdecompress - decompression routine for pglz compression method
+ *
+ * Returns the decompressed varlena.
+ */
+static struct varlena *
+pglz_cmdecompress(const struct varlena *value)
+{
+ struct varlena *result;
+ int32 rawsize;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS,
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARRAWSIZE_4B_C(value), true);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed pglz data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+}
+
+/*
+ * pglz_decompress - slice decompression routine for pglz compression method
+ *
+ * Decompresses part of the data. Returns the decompressed varlena.
+ */
+static struct varlena *
+pglz_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength)
+{
+ struct varlena *result;
+ int32 rawsize;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS,
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ slicelength, false);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed pglz data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+}
+
+/*
+ * lz4_cmcompress - compression routine for lz4 compression method
+ *
+ * Compresses source into dest using the LZ4 defaults. Returns the
+ * compressed varlena, or NULL if compression fails.
+ */
+static struct varlena *
+lz4_cmcompress(const struct varlena *value)
+{
+#ifndef HAVE_LIBLZ4
+ NO_LZ4_SUPPORT();
+#else
+ int32 valsize;
+ int32 len;
+ int32 max_size;
+ struct varlena *tmp = NULL;
+
+ valsize = VARSIZE_ANY_EXHDR(value);
+
+ /*
+ * Figure out the maximum possible size of the LZ4 output, add the bytes
+ * that will be needed for varlena overhead, and allocate that amount.
+ */
+ max_size = LZ4_compressBound(valsize);
+ tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESS);
+
+ len = LZ4_compress_default(VARDATA_ANY(value),
+ (char *) tmp + VARHDRSZ_COMPRESS,
+ valsize, max_size);
+ if (len <= 0)
+ elog(ERROR, "could not compress data with lz4");
+
+ /* data is incompressible so just free the memory and return NULL */
+ if (len > valsize)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS);
+
+ return tmp;
+#endif
+}
+
+/*
+ * lz4_cmdecompress - decompression routine for lz4 compression method
+ *
+ * Returns the decompressed varlena.
+ */
+static struct varlena *
+lz4_cmdecompress(const struct varlena *value)
+{
+#ifndef HAVE_LIBLZ4
+ NO_LZ4_SUPPORT();
+#else
+ int32 rawsize;
+ struct varlena *result;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARRAWSIZE_4B_C(value));
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed lz4 data is corrupt")));
+
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif
+}
+
+/*
+ * lz4_cmdecompress_slice - slice decompression routine for lz4 compression
+ *
+ * Decompresses part of the data. Returns the decompressed varlena.
+ */
+static struct varlena *
+lz4_cmdecompress_slice(const struct varlena *value, int32 slicelength)
+{
+#ifndef HAVE_LIBLZ4
+ NO_LZ4_SUPPORT();
+#elif LZ4_VERSION_NUMBER < 10803
+ return lz4_cmdecompress(value);
+#else
+ int32 rawsize;
+ struct varlena *result;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ slicelength,
+ slicelength);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed lz4 data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif
+}
+
+/*
+ * CompressionNameToMethod - Get compression method from compression name
+ *
+ * Search in the available built-in methods. If the compression not found
+ * in the built-in methods then return InvalidCompressionMethod.
+ */
+char
+CompressionNameToMethod(char *compression)
+{
+ if (strcmp(toast_compression[PGLZ_COMPRESSION_ID].cmname,
+ compression) == 0)
+ return PGLZ_COMPRESSION;
+ else if (strcmp(toast_compression[LZ4_COMPRESSION_ID].cmname,
+ compression) == 0)
+ {
+#ifndef HAVE_LIBLZ4
+ NO_LZ4_SUPPORT();
+#endif
+ return LZ4_COMPRESSION;
+ }
+
+ return InvalidCompressionMethod;
+}
+
+/*
+ * GetCompressionRoutines - Get compression handler routines
+ */
+const CompressionRoutine*
+GetCompressionRoutines(char method)
+{
+ return &toast_compression[CompressionMethodToId(method)];
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 9b9da0f41b..69dd9492f6 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -44,46 +44,42 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
* ----------
*/
Datum
-toast_compress_datum(Datum value)
+toast_compress_datum(Datum value, char cmethod)
{
- struct varlena *tmp;
- int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
- int32 len;
+ struct varlena *tmp = NULL;
+ int32 valsize;
+ const CompressionRoutine *cmroutine = NULL;
Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
- /*
- * No point in wasting a palloc cycle if value size is out of the allowed
- * range for compression
- */
- if (valsize < PGLZ_strategy_default->min_input_size ||
- valsize > PGLZ_strategy_default->max_input_size)
- return PointerGetDatum(NULL);
+ Assert(CompressionMethodIsValid(cmethod));
- tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
- TOAST_COMPRESS_HDRSZ);
+ /* get the handler routines for the compression method */
+ cmroutine = GetCompressionRoutines(cmethod);
+
+ /* call the actual compression function */
+ tmp = cmroutine->datum_compress((const struct varlena *) value);
+ if (!tmp)
+ return PointerGetDatum(NULL);
/*
- * We recheck the actual size even if pglz_compress() reports success,
- * because it might be satisfied with having saved as little as one byte
- * in the compressed data --- which could turn into a net loss once you
- * consider header and alignment padding. Worst case, the compressed
- * format might require three padding bytes (plus header, which is
- * included in VARSIZE(tmp)), whereas the uncompressed format would take
- * only one header byte and no padding if the value is short enough. So
- * we insist on a savings of more than 2 bytes to ensure we have a gain.
+ * We recheck the actual size even if compression reports success, because
+ * it might be satisfied with having saved as little as one byte in the
+ * compressed data --- which could turn into a net loss once you consider
+ * header and alignment padding. Worst case, the compressed format might
+ * require three padding bytes (plus header, which is included in
+ * VARSIZE(tmp)), whereas the uncompressed format would take only one
+ * header byte and no padding if the value is short enough. So we insist
+ * on a savings of more than 2 bytes to ensure we have a gain.
*/
- len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)),
- valsize,
- TOAST_COMPRESS_RAWDATA(tmp),
- PGLZ_strategy_default);
- if (len >= 0 &&
- len + TOAST_COMPRESS_HDRSZ < valsize - 2)
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+
+ if (VARSIZE(tmp) < valsize - 2)
{
- TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize);
- SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ);
/* successful compression */
+ TOAST_COMPRESS_SET_SIZE_AND_METHOD(tmp, valsize,
+ CompressionMethodToId(cmethod));
return PointerGetDatum(tmp);
}
else
diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c
index 902f59440c..a66d1113db 100644
--- a/src/backend/access/common/tupdesc.c
+++ b/src/backend/access/common/tupdesc.c
@@ -20,6 +20,7 @@
#include "postgres.h"
#include "access/htup_details.h"
+#include "access/toast_compression.h"
#include "access/tupdesc_details.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
@@ -470,6 +471,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2)
return false;
if (attr1->attcollation != attr2->attcollation)
return false;
+ if (attr1->attcompression != attr2->attcompression)
+ return false;
/* attacl, attoptions and attfdwoptions are not even present... */
}
@@ -664,6 +667,11 @@ TupleDescInitEntry(TupleDesc desc,
att->attstorage = typeForm->typstorage;
att->attcollation = typeForm->typcollation;
+ if (IsStorageCompressible(typeForm->typstorage))
+ att->attcompression = DefaultCompressionMethod;
+ else
+ att->attcompression = InvalidCompressionMethod;
+
ReleaseSysCache(tuple);
}
diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c
index bd5faf0c1f..99f5f63ea7 100644
--- a/src/backend/access/heap/heapam_handler.c
+++ b/src/backend/access/heap/heapam_handler.c
@@ -19,6 +19,7 @@
*/
#include "postgres.h"
+#include "access/detoast.h"
#include "access/genam.h"
#include "access/heapam.h"
#include "access/heaptoast.h"
@@ -26,6 +27,7 @@
#include "access/rewriteheap.h"
#include "access/syncscan.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/tsmapi.h"
#include "access/xact.h"
#include "catalog/catalog.h"
@@ -2469,6 +2471,26 @@ reform_and_rewrite_tuple(HeapTuple tuple,
{
if (TupleDescAttr(newTupDesc, i)->attisdropped)
isnull[i] = true;
+ /*
+ * Since we are rewriting the table, use this opportunity to recompress
+ * any compressed attribute with current compression method of the
+ * attribute. Basically, if the compression method of the compressed
+ * varlena is not same as current compression method of the attribute
+ * then decompress it so that if it need to be compressed then it will
+ * be compressed with the current compression method of the attribute.
+ */
+ else if (!isnull[i] && TupleDescAttr(newTupDesc, i)->attlen == -1)
+ {
+ struct varlena *new_value;
+ char cmethod;
+
+ new_value = (struct varlena *) DatumGetPointer(values[i]);
+ cmethod = toast_get_compression_method(new_value);
+
+ if (IsValidCompression(cmethod) &&
+ TupleDescAttr(newTupDesc, i)->attcompression != cmethod)
+ values[i] = PointerGetDatum(detoast_attr(new_value));
+ }
}
copiedTuple = heap_form_tuple(newTupDesc, values, isnull);
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index fb36151ce5..53f78f9c3e 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -54,6 +54,7 @@ toast_tuple_init(ToastTupleContext *ttc)
ttc->ttc_attr[i].tai_colflags = 0;
ttc->ttc_attr[i].tai_oldexternal = NULL;
+ ttc->ttc_attr[i].tai_compression = att->attcompression;
if (ttc->ttc_oldvalues != NULL)
{
@@ -226,9 +227,11 @@ void
toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
{
Datum *value = &ttc->ttc_values[attribute];
- Datum new_value = toast_compress_datum(*value);
+ Datum new_value;
ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+ new_value = toast_compress_datum(*value, attr->tai_compression);
+
if (DatumGetPointer(new_value) != NULL)
{
/* successful compression */
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index 6f615e6622..af0c0aa77e 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -21,6 +21,7 @@
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "access/xlog_internal.h"
#include "bootstrap/bootstrap.h"
@@ -731,6 +732,10 @@ DefineAttr(char *name, char *type, int attnum, int nullness)
attrtypes[attnum]->attcacheoff = -1;
attrtypes[attnum]->atttypmod = -1;
attrtypes[attnum]->attislocal = true;
+ if (IsStorageCompressible(attrtypes[attnum]->attstorage))
+ attrtypes[attnum]->attcompression = DefaultCompressionMethod;
+ else
+ attrtypes[attnum]->attcompression = InvalidCompressionMethod;
if (nullness == BOOTCOL_NULL_FORCE_NOT_NULL)
{
diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl
index b159958112..9586c29ad0 100644
--- a/src/backend/catalog/genbki.pl
+++ b/src/backend/catalog/genbki.pl
@@ -906,6 +906,9 @@ sub morph_row_for_pgattr
$row->{attcollation} =
$type->{typcollation} ne '0' ? $C_COLLATION_OID : 0;
+ $row->{attcompression} =
+ $type->{typstorage} ne 'p' && $type->{typstorage} ne 'e' ? 'p' : '\0';
+
if (defined $attr->{forcenotnull})
{
$row->{attnotnull} = 't';
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 693a94107d..7033ba9bed 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -36,6 +36,7 @@
#include "access/sysattr.h"
#include "access/table.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/transam.h"
#include "access/xact.h"
#include "access/xlog.h"
@@ -789,6 +790,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel,
slot[slotCount]->tts_values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(attrs->attislocal);
slot[slotCount]->tts_values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(attrs->attinhcount);
slot[slotCount]->tts_values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(attrs->attcollation);
+ slot[slotCount]->tts_values[Anum_pg_attribute_attcompression - 1] = CharGetDatum(attrs->attcompression);
if (attoptions && attoptions[natts] != (Datum) 0)
slot[slotCount]->tts_values[Anum_pg_attribute_attoptions - 1] = attoptions[natts];
else
@@ -1716,6 +1718,8 @@ RemoveAttributeById(Oid relid, AttrNumber attnum)
/* Unset this so no one tries to look up the generation expression */
attStruct->attgenerated = '\0';
+ attStruct->attcompression = InvalidCompressionMethod;
+
/*
* Change the column name to something that isn't likely to conflict
*/
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4ef61b5efd..397d70d226 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -348,6 +348,7 @@ ConstructTupleDescriptor(Relation heapRelation,
to->attbyval = from->attbyval;
to->attstorage = from->attstorage;
to->attalign = from->attalign;
+ to->attcompression = from->attcompression;
}
else
{
diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c
index d7b806020d..933a0734d1 100644
--- a/src/backend/catalog/toasting.c
+++ b/src/backend/catalog/toasting.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include "access/heapam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "catalog/binary_upgrade.h"
#include "catalog/catalog.h"
@@ -220,6 +221,11 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
TupleDescAttr(tupdesc, 1)->attstorage = TYPSTORAGE_PLAIN;
TupleDescAttr(tupdesc, 2)->attstorage = TYPSTORAGE_PLAIN;
+ /* Toast field should not be compressed */
+ TupleDescAttr(tupdesc, 0)->attcompression = InvalidCompressionMethod;
+ TupleDescAttr(tupdesc, 1)->attcompression = InvalidCompressionMethod;
+ TupleDescAttr(tupdesc, 2)->attcompression = InvalidCompressionMethod;
+
/*
* Toast tables for regular relations go in pg_toast; those for temp
* relations go into the per-backend temp-toast-table namespace.
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 8745c1e722..e20dd5f63c 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -24,6 +24,7 @@
#include "access/relscan.h"
#include "access/sysattr.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/catalog.h"
@@ -562,6 +563,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx,
static List *GetParentedForeignKeyRefs(Relation partition);
static void ATDetachCheckNoForeignKeyRefs(Relation partition);
static void ATExecAlterCollationRefreshVersion(Relation rel, List *coll);
+static char GetAttributeCompression(Form_pg_attribute att, char *compression);
/* ----------------------------------------------------------------
@@ -855,6 +857,18 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
if (colDef->generated)
attr->attgenerated = colDef->generated;
+
+ /*
+ * lookup attribute's compression method and store it in the
+ * attr->attcompression.
+ */
+ if (relkind == RELKIND_RELATION ||
+ relkind == RELKIND_PARTITIONED_TABLE ||
+ relkind == RELKIND_MATVIEW)
+ attr->attcompression =
+ GetAttributeCompression(attr, colDef->compression);
+ else
+ attr->attcompression = InvalidCompressionMethod;
}
/*
@@ -2399,6 +2413,22 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
storage_name(def->storage),
storage_name(attribute->attstorage))));
+ /* Copy/check compression parameter */
+ if (CompressionMethodIsValid(attribute->attcompression))
+ {
+ const char *compression = GetCompressionMethodName(
+ attribute->attcompression);
+
+ if (def->compression == NULL)
+ def->compression = pstrdup(compression);
+ else if (strcmp(def->compression, compression) != 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("column \"%s\" has a compression method conflict",
+ attributeName),
+ errdetail("%s versus %s", def->compression, compression)));
+ }
+
def->inhcount++;
/* Merge of NOT NULL constraints = OR 'em together */
def->is_not_null |= attribute->attnotnull;
@@ -2433,6 +2463,11 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
def->collOid = attribute->attcollation;
def->constraints = NIL;
def->location = -1;
+ if (CompressionMethodIsValid(attribute->attcompression))
+ def->compression = pstrdup(GetCompressionMethodName(
+ attribute->attcompression));
+ else
+ def->compression = NULL;
inhSchema = lappend(inhSchema, def);
newattmap->attnums[parent_attno - 1] = ++child_attno;
}
@@ -2678,6 +2713,19 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
storage_name(def->storage),
storage_name(newdef->storage))));
+ /* Copy compression parameter */
+ if (def->compression == NULL)
+ def->compression = newdef->compression;
+ else if (newdef->compression != NULL)
+ {
+ if (strcmp(def->compression, newdef->compression) != 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("column \"%s\" has a compression method conflict",
+ attributeName),
+ errdetail("%s versus %s", def->compression, newdef->compression)));
+ }
+
/* Mark the column as locally defined */
def->is_local = true;
/* Merge of NOT NULL constraints = OR 'em together */
@@ -6380,6 +6428,18 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
attribute.attislocal = colDef->is_local;
attribute.attinhcount = colDef->inhcount;
attribute.attcollation = collOid;
+
+ /*
+ * lookup attribute's compression method and store it in the
+ * attr->attcompression.
+ */
+ if (rel->rd_rel->relkind == RELKIND_RELATION ||
+ rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ attribute.attcompression = GetAttributeCompression(&attribute,
+ colDef->compression);
+ else
+ attribute.attcompression = InvalidCompressionMethod;
+
/* attribute.attacl is handled by InsertPgAttributeTuples() */
ReleaseSysCache(typeTuple);
@@ -11899,6 +11959,23 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
ReleaseSysCache(typeTuple);
+ /* Setup attribute compression */
+ if (rel->rd_rel->relkind == RELKIND_RELATION ||
+ rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ {
+ /*
+ * No compression for plain/external storage, otherwise, default
+ * compression method if it is not already set, refer comments atop
+ * attcompression parameter in pg_attribute.h.
+ */
+ if (!IsStorageCompressible(tform->typstorage))
+ attTup->attcompression = InvalidCompressionMethod;
+ else if (!CompressionMethodIsValid(attTup->attcompression))
+ attTup->attcompression = DefaultCompressionMethod;
+ }
+ else
+ attTup->attcompression = InvalidCompressionMethod;
+
CatalogTupleUpdate(attrelation, &heapTup->t_self, heapTup);
table_close(attrelation, RowExclusiveLock);
@@ -17743,3 +17820,36 @@ ATExecAlterCollationRefreshVersion(Relation rel, List *coll)
index_update_collation_versions(rel->rd_id, get_collation_oid(coll, false));
CacheInvalidateRelcache(rel);
}
+
+/*
+ * resolve column compression specification to compression method.
+ */
+static char
+GetAttributeCompression(Form_pg_attribute att, char *compression)
+{
+ char typstorage = get_typstorage(att->atttypid);
+ char cmethod;
+
+ /*
+ * No compression for plain/external storage, refer comments atop
+ * attcompression parameter in pg_attribute.h
+ */
+ if (!IsStorageCompressible(typstorage))
+ {
+ if (compression == NULL)
+ return InvalidCompressionMethod;
+
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("column data type %s does not support compression",
+ format_type_be(att->atttypid))));
+ }
+
+ /* fallback to default compression if it's not specified */
+ if (compression == NULL)
+ return DefaultCompressionMethod;
+
+ cmethod = CompressionNameToMethod(compression);
+
+ return cmethod;
+}
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 2993ba43e3..631ba1f193 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,9 +37,11 @@
#include "postgres.h"
+#include "access/detoast.h"
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "catalog/catalog.h"
#include "commands/trigger.h"
@@ -2036,6 +2038,119 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate,
return slot;
}
+/*
+ * Compare the compression method of each compressed value with the
+ * compression method of the target attribute. If the compression method
+ * of the compressed value is not same as the target attribute then decompress
+ * the value. If any of the value need to be decompressed then we need to
+ * store that into a new slot.
+ */
+TupleTableSlot *
+CompareCompressionMethodAndDecompress(TupleTableSlot *slot,
+ TupleTableSlot **outslot,
+ TupleDesc targetTupDesc)
+{
+ int i;
+ int attnum;
+ int natts = slot->tts_tupleDescriptor->natts;
+ bool decompressed_any = false;
+ bool slot_tup_deformed = false;
+ TupleDesc tupleDesc = slot->tts_tupleDescriptor;
+ TupleTableSlot *newslot = *outslot;
+
+ if (natts == 0)
+ return slot;
+
+ /*
+ * Loop over all the attributes in the tuple and check if any attribute is
+ * compressed and its compression method is not same as the target
+ * atrribute's compression method then decompress it.
+ */
+ for (i = 0; i < natts; i++)
+ {
+ attnum = tupleDesc->attrs[i].attnum;
+
+ if (TupleDescAttr(tupleDesc, i)->attlen == -1)
+ {
+ struct varlena *new_value;
+ char cmethod;
+
+ /* fetch the values of all the attribute, if not already done */
+ if (!slot_tup_deformed)
+ {
+ slot_getallattrs(slot);
+ slot_tup_deformed = true;
+ }
+
+ /* nothing to be done, if the value is null */
+ if (slot->tts_isnull[attnum - 1])
+ continue;
+
+ new_value = (struct varlena *)
+ DatumGetPointer(slot->tts_values[attnum - 1]);
+
+ /*
+ * Get the compression method stored in the toast header and
+ * compare it with the compression method of the target.
+ */
+ cmethod = toast_get_compression_method(new_value);
+ if (IsValidCompression(cmethod) &&
+ targetTupDesc->attrs[i].attcompression != cmethod)
+ {
+ if (!decompressed_any)
+ {
+ /*
+ * If the caller has passed an invalid slot then create a
+ * new slot. Otherwise, just clear the existing tuple from
+ * the slot. This slot should be stored by the caller so
+ * that it can be reused for decompressing the subsequent
+ * tuples.
+ */
+ if (newslot == NULL)
+ newslot = MakeSingleTupleTableSlot(tupleDesc,
+ slot->tts_ops);
+ else
+ ExecClearTuple(newslot);
+
+ /* copy the value/null array to the new slot */
+ memcpy(newslot->tts_values, slot->tts_values,
+ natts * sizeof(Datum));
+ memcpy(newslot->tts_isnull, slot->tts_isnull,
+ natts * sizeof(bool));
+
+ }
+
+ /* detoast the value and store into the new slot */
+ new_value = detoast_attr(new_value);
+ newslot->tts_values[attnum - 1] = PointerGetDatum(new_value);
+ decompressed_any = true;
+ }
+ }
+ }
+
+ /*
+ * If we have decompressed any of the fields then return the new slot
+ * otherwise return the original slot.
+ */
+ if (decompressed_any)
+ {
+ ExecStoreVirtualTuple(newslot);
+
+ /*
+ * If the original slot was materialized then materialize the new slot
+ * as well.
+ */
+ if (TTS_SHOULDFREE(slot))
+ ExecMaterializeSlot(newslot);
+
+ *outslot = newslot;
+
+ return newslot;
+ }
+ else
+ return slot;
+}
+
/* ----------------------------------------------------------------
* ExecModifyTable
*
@@ -2244,6 +2359,15 @@ ExecModifyTable(PlanState *pstate)
slot = ExecFilterJunk(junkfilter, slot);
}
+ /*
+ * Compare the compression method of the compressed attribute in the
+ * source tuple with target attribute and if those are different then
+ * decompress those attributes.
+ */
+ slot = CompareCompressionMethodAndDecompress(slot,
+ &node->mt_decompress_tuple_slot,
+ resultRelInfo->ri_RelationDesc->rd_att);
+
switch (operation)
{
case CMD_INSERT:
@@ -2876,6 +3000,10 @@ ExecEndModifyTable(ModifyTableState *node)
ExecDropSingleTupleTableSlot(node->mt_root_tuple_slot);
}
+ /* release the slot used for decompressing the tuple */
+ if (node->mt_decompress_tuple_slot)
+ ExecDropSingleTupleTableSlot(node->mt_decompress_tuple_slot);
+
/*
* Free the exprcontext
*/
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index da91cbd2b1..0d3b923a38 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -2988,6 +2988,7 @@ _copyColumnDef(const ColumnDef *from)
COPY_STRING_FIELD(colname);
COPY_NODE_FIELD(typeName);
+ COPY_STRING_FIELD(compression);
COPY_SCALAR_FIELD(inhcount);
COPY_SCALAR_FIELD(is_local);
COPY_SCALAR_FIELD(is_not_null);
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index c2d73626fc..f3592003da 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -2599,6 +2599,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b)
{
COMPARE_STRING_FIELD(colname);
COMPARE_NODE_FIELD(typeName);
+ COMPARE_STRING_FIELD(compression);
COMPARE_SCALAR_FIELD(inhcount);
COMPARE_SCALAR_FIELD(is_local);
COMPARE_SCALAR_FIELD(is_not_null);
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 49357ac5c2..38226530c6 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -3897,6 +3897,8 @@ raw_expression_tree_walker(Node *node,
if (walker(coldef->typeName, context))
return true;
+ if (walker(coldef->compression, context))
+ return true;
if (walker(coldef->raw_default, context))
return true;
if (walker(coldef->collClause, context))
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6493a03ff8..75343b8aaf 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -2876,6 +2876,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node)
WRITE_STRING_FIELD(colname);
WRITE_NODE_FIELD(typeName);
+ WRITE_STRING_FIELD(compression);
WRITE_INT_FIELD(inhcount);
WRITE_BOOL_FIELD(is_local);
WRITE_BOOL_FIELD(is_not_null);
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 2ae41f8f4d..f8bc11d28b 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -596,6 +596,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound
%type <defelt> hash_partbound_elem
+%type <str> optColumnCompression
+
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -631,9 +633,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT
- CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE
- CROSS CSV CUBE CURRENT_P
+ COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
+ COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
@@ -3429,11 +3431,12 @@ TypedTableElement:
| TableConstraint { $$ = $1; }
;
-columnDef: ColId Typename create_generic_options ColQualList
+columnDef: ColId Typename optColumnCompression create_generic_options ColQualList
{
ColumnDef *n = makeNode(ColumnDef);
n->colname = $1;
n->typeName = $2;
+ n->compression = $3;
n->inhcount = 0;
n->is_local = true;
n->is_not_null = false;
@@ -3442,8 +3445,8 @@ columnDef: ColId Typename create_generic_options ColQualList
n->raw_default = NULL;
n->cooked_default = NULL;
n->collOid = InvalidOid;
- n->fdwoptions = $3;
- SplitColQualList($4, &n->constraints, &n->collClause,
+ n->fdwoptions = $4;
+ SplitColQualList($5, &n->constraints, &n->collClause,
yyscanner);
n->location = @1;
$$ = (Node *)n;
@@ -3488,6 +3491,14 @@ columnOptions: ColId ColQualList
}
;
+optColumnCompression:
+ COMPRESSION name
+ {
+ $$ = $2;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
ColQualList:
ColQualList ColConstraint { $$ = lappend($1, $2); }
| /*EMPTY*/ { $$ = NIL; }
@@ -3718,6 +3729,7 @@ TableLikeOption:
| INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; }
| STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; }
| STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; }
+ | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; }
| ALL { $$ = CREATE_TABLE_LIKE_ALL; }
;
@@ -15304,6 +15316,7 @@ unreserved_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -15824,6 +15837,7 @@ bare_label_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONCURRENTLY
| CONFIGURATION
| CONFLICT
diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c
index 75266caeb4..681151e32c 100644
--- a/src/backend/parser/parse_utilcmd.c
+++ b/src/backend/parser/parse_utilcmd.c
@@ -31,6 +31,7 @@
#include "access/relation.h"
#include "access/reloptions.h"
#include "access/table.h"
+#include "access/toast_compression.h"
#include "catalog/dependency.h"
#include "catalog/heap.h"
#include "catalog/index.h"
@@ -1082,6 +1083,14 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla
else
def->storage = 0;
+ /* Likewise, copy compression if requested */
+ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION) != 0
+ && CompressionMethodIsValid(attribute->attcompression))
+ def->compression =
+ pstrdup(GetCompressionMethodName(attribute->attcompression));
+ else
+ def->compression = NULL;
+
/* Likewise, copy comment if requested */
if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) &&
(comment = GetComment(attribute->attrelid,
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 479ed9ae54..187e55bd4e 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -18,6 +18,7 @@
#include <limits.h>
#include "access/detoast.h"
+#include "access/toast_compression.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
#include "common/hashfn.h"
@@ -5299,6 +5300,46 @@ pg_column_size(PG_FUNCTION_ARGS)
PG_RETURN_INT32(result);
}
+/*
+ * Return the compression method stored in the compressed attribute. Return
+ * NULL for non varlena type or uncompressed data.
+ */
+Datum
+pg_column_compression(PG_FUNCTION_ARGS)
+{
+ Datum value = PG_GETARG_DATUM(0);
+ int typlen;
+ char method;
+
+ /* On first call, get the input type's typlen, and save at *fn_extra */
+ if (fcinfo->flinfo->fn_extra == NULL)
+ {
+ /* Lookup the datatype of the supplied argument */
+ Oid argtypeid = get_fn_expr_argtype(fcinfo->flinfo, 0);
+
+ typlen = get_typlen(argtypeid);
+ if (typlen == 0) /* should not happen */
+ elog(ERROR, "cache lookup failed for type %u", argtypeid);
+
+ fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt,
+ sizeof(int));
+ *((int *) fcinfo->flinfo->fn_extra) = typlen;
+ }
+ else
+ typlen = *((int *) fcinfo->flinfo->fn_extra);
+
+ if (typlen != -1)
+ PG_RETURN_NULL();
+
+ method = toast_get_compression_method(
+ (struct varlena *) DatumGetPointer(value));
+
+ if (!CompressionMethodIsValid(method))
+ PG_RETURN_NULL();
+ else
+ PG_RETURN_TEXT_P(cstring_to_text(GetCompressionMethodName(method)));
+}
+
/*
* string_agg - Concatenates values and returns string.
*
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 99ace71a2a..f54f285f38 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -161,6 +161,7 @@ typedef struct _dumpOptions
int no_subscriptions;
int no_synchronized_snapshots;
int no_unlogged_table_data;
+ int no_toast_compression;
int serializable_deferrable;
int disable_triggers;
int outputNoTableam;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 3fd7f48605..be5d369efa 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -387,6 +387,7 @@ main(int argc, char **argv)
{"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
+ {"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-sync", no_argument, NULL, 7},
{"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1},
{"rows-per-insert", required_argument, NULL, 10},
@@ -1048,6 +1049,7 @@ help(const char *progname)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
+ printf(_(" --no-toast-compression do not dump toast compression methods\n"));
printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
printf(_(" --no-unlogged-table-data do not dump unlogged table data\n"));
@@ -8626,6 +8628,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
{
DumpOptions *dopt = fout->dopt;
PQExpBuffer q = createPQExpBuffer();
+ bool createWithCompression;
for (int i = 0; i < numTables; i++)
{
@@ -8711,6 +8714,15 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
appendPQExpBufferStr(q,
"'' AS attidentity,\n");
+ createWithCompression = (fout->remoteVersion >= 140000);
+
+ if (createWithCompression)
+ appendPQExpBuffer(q,
+ "a.attcompression AS attcompression,\n");
+ else
+ appendPQExpBuffer(q,
+ "NULL AS attcmname,\n");
+
if (fout->remoteVersion >= 110000)
appendPQExpBufferStr(q,
"CASE WHEN a.atthasmissing AND NOT a.attisdropped "
@@ -8756,6 +8768,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid));
tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *));
tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *));
+ tbinfo->attcompression = (char *) pg_malloc(ntups * sizeof(char *));
tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *));
@@ -8784,6 +8797,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, PQfnumber(res, "attcollation")));
tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attfdwoptions")));
tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attmissingval")));
+ tbinfo->attcompression[j] = *(PQgetvalue(res, j, PQfnumber(res, "attcompression")));
tbinfo->attrdefs[j] = NULL; /* fix below */
if (PQgetvalue(res, j, PQfnumber(res, "atthasdef"))[0] == 't')
hasdefaults = true;
@@ -15900,6 +15914,31 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
tbinfo->atttypnames[j]);
}
+ /*
+ * Attribute compression
+ */
+ if (!dopt->no_toast_compression &&
+ tbinfo->attcompression[j] &&
+ tbinfo->attcompression[j] != '\0')
+ {
+ char *cmname;
+
+ switch (tbinfo->attcompression[j])
+ {
+ case 'p':
+ cmname = "pglz";
+ break;
+ case 'l':
+ cmname = "lz4";
+ break;
+ default:
+ cmname = NULL;
+ }
+
+ if (cmname != NULL)
+ appendPQExpBuffer(q, " COMPRESSION %s", cmname);
+ }
+
if (print_default)
{
if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED)
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 0a2213fb06..d956b035f3 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -326,6 +326,7 @@ typedef struct _tableInfo
char *partbound; /* partition bound definition */
bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */
char *amname; /* relation access method */
+ char *attcompression; /* per-attribute current compression method */
/*
* Stuff computed only for dumpable tables.
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 737e46464a..bc91bb12ac 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -2284,9 +2284,9 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text,\E\n
- \s+\Qcol3 text,\E\n
- \s+\Qcol4 text,\E\n
+ \s+\Qcol2 text COMPRESSION\E\D*,\n
+ \s+\Qcol3 text COMPRESSION\E\D*,\n
+ \s+\Qcol4 text COMPRESSION\E\D*,\n
\s+\QCONSTRAINT test_table_col1_check CHECK ((col1 <= 1000))\E\n
\Q)\E\n
\QWITH (autovacuum_enabled='false', fillfactor='80');\E\n/xm,
@@ -2326,7 +2326,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_second_table (\E
\n\s+\Qcol1 integer,\E
- \n\s+\Qcol2 text\E
+ \n\s+\Qcol2 text COMPRESSION\E\D*
\n\);
/xm,
like =>
@@ -2441,7 +2441,7 @@ my %tests = (
\n\s+\Qcol1 integer,\E
\n\s+\Qcol2 boolean,\E
\n\s+\Qcol3 boolean,\E
- \n\s+\Qcol4 bit(5),\E
+ \n\s+\Qcol4 bit(5) COMPRESSION\E\D*,
\n\s+\Qcol5 double precision\E
\n\);
/xm,
@@ -2459,7 +2459,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table_identity (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text\E\n
+ \s+\Qcol2 text COMPRESSION\E\D*\n
\);
.*
\QALTER TABLE dump_test.test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 20af5a92b4..b284113d55 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -1459,7 +1459,7 @@ describeOneTableDetails(const char *schemaname,
bool printTableInitialized = false;
int i;
char *view_def = NULL;
- char *headers[11];
+ char *headers[12];
PQExpBufferData title;
PQExpBufferData tmpbuf;
int cols;
@@ -1475,7 +1475,8 @@ describeOneTableDetails(const char *schemaname,
fdwopts_col = -1,
attstorage_col = -1,
attstattarget_col = -1,
- attdescr_col = -1;
+ attdescr_col = -1,
+ attcompression_col = -1;
int numrows;
struct
{
@@ -1892,6 +1893,17 @@ describeOneTableDetails(const char *schemaname,
appendPQExpBufferStr(&buf, ",\n a.attstorage");
attstorage_col = cols++;
+ /* compresssion info */
+ if (pset.sversion >= 140000 &&
+ !pset.hide_compression &&
+ (tableinfo.relkind == RELKIND_RELATION ||
+ tableinfo.relkind == RELKIND_PARTITIONED_TABLE ||
+ tableinfo.relkind == RELKIND_MATVIEW))
+ {
+ appendPQExpBufferStr(&buf, ",\n a.attcompression AS attcompression");
+ attcompression_col = cols++;
+ }
+
/* stats target, if relevant to relkind */
if (tableinfo.relkind == RELKIND_RELATION ||
tableinfo.relkind == RELKIND_INDEX ||
@@ -2018,6 +2030,8 @@ describeOneTableDetails(const char *schemaname,
headers[cols++] = gettext_noop("FDW options");
if (attstorage_col >= 0)
headers[cols++] = gettext_noop("Storage");
+ if (attcompression_col >= 0)
+ headers[cols++] = gettext_noop("Compression");
if (attstattarget_col >= 0)
headers[cols++] = gettext_noop("Stats target");
if (attdescr_col >= 0)
@@ -2097,6 +2111,19 @@ describeOneTableDetails(const char *schemaname,
false, false);
}
+ /* Column compression. */
+ if (attcompression_col >= 0)
+ {
+ char *compression = PQgetvalue(res, i, attcompression_col);
+
+ /* these strings are literal in our syntax, so not translated. */
+ printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
+ (compression[0] == 'l' ? "lz4" :
+ (compression[0] == '\0' ? "" :
+ "???"))),
+ false, false);
+ }
+
/* Statistics target, if the relkind supports this feature */
if (attstattarget_col >= 0)
printTableAddCell(&cont, PQgetvalue(res, i, attstattarget_col),
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index daa5081eac..99a59470c5 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -372,6 +372,8 @@ helpVariables(unsigned short int pager)
" true if last query failed, else false\n"));
fprintf(output, _(" FETCH_COUNT\n"
" the number of result rows to fetch and display at a time (0 = unlimited)\n"));
+ fprintf(output, _(" HIDE_TOAST_COMPRESSION\n"
+ " if set, compression methods are not displayed\n"));
fprintf(output, _(" HIDE_TABLEAM\n"
" if set, table access methods are not displayed\n"));
fprintf(output, _(" HISTCONTROL\n"
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index d65990059d..83f2e6f254 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -134,6 +134,7 @@ typedef struct _psqlSettings
bool quiet;
bool singleline;
bool singlestep;
+ bool hide_compression;
bool hide_tableam;
int fetch_count;
int histsize;
diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c
index 780479c8d7..110906a4e9 100644
--- a/src/bin/psql/startup.c
+++ b/src/bin/psql/startup.c
@@ -1159,6 +1159,13 @@ show_context_hook(const char *newval)
return true;
}
+static bool
+hide_compression_hook(const char *newval)
+{
+ return ParseVariableBool(newval, "HIDE_TOAST_COMPRESSION",
+ &pset.hide_compression);
+}
+
static bool
hide_tableam_hook(const char *newval)
{
@@ -1227,6 +1234,9 @@ EstablishVariableSpace(void)
SetVariableHooks(pset.vars, "SHOW_CONTEXT",
show_context_substitute_hook,
show_context_hook);
+ SetVariableHooks(pset.vars, "HIDE_TOAST_COMPRESSION",
+ bool_substitute_hook,
+ hide_compression_hook);
SetVariableHooks(pset.vars, "HIDE_TABLEAM",
bool_substitute_hook,
hide_tableam_hook);
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index 0adf53c77b..9a428aae2d 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -89,4 +89,12 @@ extern Size toast_raw_datum_size(Datum value);
*/
extern Size toast_datum_size(Datum value);
+/* ----------
+ * toast_get_compression_method -
+ *
+ * Return the compression method from the compressed value
+ * ----------
+ */
+extern char toast_get_compression_method(struct varlena *attr);
+
#endif /* DETOAST_H */
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
new file mode 100644
index 0000000000..38800cb97a
--- /dev/null
+++ b/src/include/access/toast_compression.h
@@ -0,0 +1,119 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_compression.h
+ * Functions for toast compression.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/toast_compression.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TOAST_COMPRESSION_H
+#define TOAST_COMPRESSION_H
+
+#include "postgres.h"
+
+/*
+ * Built-in compression methods. pg_attribute will store this in the
+ * attcompression column.
+ */
+#define PGLZ_COMPRESSION 'p'
+#define LZ4_COMPRESSION 'l'
+
+#define InvalidCompressionMethod '\0'
+#define CompressionMethodIsValid(cm) ((bool) ((cm) != InvalidCompressionMethod))
+
+/*
+ * Built-in compression method-id. The toast compression header will store
+ * this in the first 2 bits of the raw length. These built-in compression
+ * method-id are directly mapped to the built-in compression methods.
+ */
+typedef enum CompressionId
+{
+ PGLZ_COMPRESSION_ID = 0,
+ LZ4_COMPRESSION_ID = 1
+} CompressionId;
+
+/* use default compression method if it is not specified. */
+#define DefaultCompressionMethod PGLZ_COMPRESSION
+#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod)
+
+#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \
+ (storage) != TYPSTORAGE_EXTERNAL)
+
+/* compression handler routines */
+typedef struct varlena *(*cmcompress_function) (const struct varlena *value);
+typedef struct varlena *(*cmdecompress_function) (const struct varlena *value);
+typedef struct varlena *(*cmdecompress_slice_function)
+ (const struct varlena *value, int32 slicelength);
+
+/*
+ * API struct for a compression routines.
+ *
+ * 'cmname' - name of the compression method
+ * 'datum_compress' - varlena compression function.
+ * 'datum_decompress' - varlena decompression function.
+ * 'datum_decompress_slice' - varlena slice decompression functions.
+ */
+typedef struct CompressionRoutine
+{
+ char cmname[64];
+ cmcompress_function datum_compress;
+ cmdecompress_function datum_decompress;
+ cmdecompress_slice_function datum_decompress_slice;
+} CompressionRoutine;
+
+extern char CompressionNameToMethod(char *compression);
+extern const CompressionRoutine *GetCompressionRoutines(char method);
+
+/*
+ * CompressionMethodToId - Convert compression method to compression id.
+ *
+ * For more details refer comment atop CompressionId in toast_compression.h
+ */
+static inline CompressionId
+CompressionMethodToId(char method)
+{
+ switch (method)
+ {
+ case PGLZ_COMPRESSION:
+ return PGLZ_COMPRESSION_ID;
+ case LZ4_COMPRESSION:
+ return LZ4_COMPRESSION_ID;
+ default:
+ elog(ERROR, "invalid compression method %c", method);
+ }
+}
+
+/*
+ * CompressionIdToMethod - Convert compression id to compression method
+ *
+ * For more details refer comment atop CompressionId in toast_compression.h
+ */
+static inline Oid
+CompressionIdToMethod(CompressionId cmid)
+{
+ switch (cmid)
+ {
+ case PGLZ_COMPRESSION_ID:
+ return PGLZ_COMPRESSION;
+ case LZ4_COMPRESSION_ID:
+ return LZ4_COMPRESSION;
+ default:
+ elog(ERROR, "invalid compression method id %d", cmid);
+ }
+}
+
+/*
+ * GetCompressionMethodName - Get compression method name
+ */
+static inline const char*
+GetCompressionMethodName(char method)
+{
+ return GetCompressionRoutines(method)->cmname;
+}
+
+
+#endif /* TOAST_COMPRESSION_H */
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index a9a6d644bc..05104ce237 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -32,6 +32,7 @@ typedef struct
struct varlena *tai_oldexternal;
int32 tai_size;
uint8 tai_colflags;
+ char tai_compression;
} ToastAttrInfo;
/*
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index cedfb890d8..284a15761a 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -12,6 +12,7 @@
#ifndef TOAST_INTERNALS_H
#define TOAST_INTERNALS_H
+#include "access/toast_compression.h"
#include "storage/lockdefs.h"
#include "utils/relcache.h"
#include "utils/snapshot.h"
@@ -22,22 +23,23 @@
typedef struct toast_compress_header
{
int32 vl_len_; /* varlena header (do not touch directly!) */
- int32 rawsize;
+ uint32 tcinfo; /* 2 bits for compression method and 30 bits
+ * rawsize */
} toast_compress_header;
/*
* Utilities for manipulation of header information for compressed
* toast entries.
*/
-#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header))
-#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize)
-#define TOAST_COMPRESS_SIZE(ptr) ((int32) VARSIZE_ANY(ptr) - TOAST_COMPRESS_HDRSZ)
-#define TOAST_COMPRESS_RAWDATA(ptr) \
- (((char *) (ptr)) + TOAST_COMPRESS_HDRSZ)
-#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \
- (((toast_compress_header *) (ptr))->rawsize = (len))
+#define TOAST_COMPRESS_METHOD(ptr) (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_RAWSIZE_BITS)
+#define TOAST_COMPRESS_SET_SIZE_AND_METHOD(ptr, len, cm_method) \
+ do { \
+ Assert((len) > 0 && (len) <= VARLENA_RAWSIZE_MASK); \
+ Assert((cm_method) >= PGLZ_COMPRESSION_ID && (cm_method) <= LZ4_COMPRESSION_ID); \
+ ((toast_compress_header *) (ptr))->tcinfo = ((len) | (cm_method) << VARLENA_RAWSIZE_BITS); \
+ } while (0)
-extern Datum toast_compress_datum(Datum value);
+extern Datum toast_compress_datum(Datum value, char cmethod);
extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h
index 3db42abf08..560f8f00bb 100644
--- a/src/include/catalog/pg_attribute.h
+++ b/src/include/catalog/pg_attribute.h
@@ -160,6 +160,12 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75,
/* attribute's collation, if any */
Oid attcollation BKI_LOOKUP_OPT(pg_collation);
+ /*
+ * compression method. Must be InvalidCompressionMethod if and only if
+ * typstorage is 'plain' or 'external'.
+ */
+ char attcompression BKI_DEFAULT('\0');
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* NOTE: The following fields are not present in tuple descriptors. */
@@ -187,7 +193,7 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75,
* can access fields beyond attcollation except in a real tuple!
*/
#define ATTRIBUTE_FIXED_PART_SIZE \
- (offsetof(FormData_pg_attribute,attcollation) + sizeof(Oid))
+ (offsetof(FormData_pg_attribute,attcompression) + sizeof(char))
/* ----------------
* Form_pg_attribute corresponds to a pointer to a tuple with
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index c7619f8cd3..91c1c5b720 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7099,6 +7099,10 @@
descr => 'bytes required to store the value, perhaps with compression',
proname => 'pg_column_size', provolatile => 's', prorettype => 'int4',
proargtypes => 'any', prosrc => 'pg_column_size' },
+{ oid => '2121',
+ descr => 'compression method for the compressed datum',
+ proname => 'pg_column_compression', provolatile => 's', prorettype => 'text',
+ proargtypes => 'any', prosrc => 'pg_column_compression' },
{ oid => '2322',
descr => 'total disk space usage for the specified tablespace',
proname => 'pg_tablespace_size', provolatile => 'v', prorettype => 'int8',
diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h
index 071e363d54..6495162a33 100644
--- a/src/include/executor/executor.h
+++ b/src/include/executor/executor.h
@@ -621,5 +621,7 @@ extern void CheckCmdReplicaIdentity(Relation rel, CmdType cmd);
extern void CheckSubscriptionRelkind(char relkind, const char *nspname,
const char *relname);
-
+extern TupleTableSlot *CompareCompressionMethodAndDecompress(TupleTableSlot *slot,
+ TupleTableSlot **outslot,
+ TupleDesc targetTupDesc);
#endif /* EXECUTOR_H */
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index e31ad6204e..e388336e02 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1185,6 +1185,12 @@ typedef struct ModifyTableState
*/
TupleTableSlot *mt_root_tuple_slot;
+ /*
+ * Slot for storing the modified tuple, in case the target attribute's
+ * compression method doesn't match that of the source table.
+ */
+ TupleTableSlot *mt_decompress_tuple_slot;
+
/* Tuple-routing support info */
struct PartitionTupleRouting *mt_partition_tuple_routing;
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ce54f76610..56e24529f1 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -646,6 +646,7 @@ typedef struct ColumnDef
NodeTag type;
char *colname; /* name of column */
TypeName *typeName; /* type of column */
+ char *compression; /* compression method for column */
int inhcount; /* number of times column is inherited */
bool is_local; /* column has local (non-inherited) def'n */
bool is_not_null; /* NOT NULL constraint specified? */
@@ -685,6 +686,7 @@ typedef enum TableLikeOption
CREATE_TABLE_LIKE_INDEXES = 1 << 5,
CREATE_TABLE_LIKE_STATISTICS = 1 << 6,
CREATE_TABLE_LIKE_STORAGE = 1 << 7,
+ CREATE_TABLE_LIKE_COMPRESSION = 1 << 8,
CREATE_TABLE_LIKE_ALL = PG_INT32_MAX
} TableLikeOption;
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 28083aaac9..ca1f950cbe 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..6007d72a73 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -346,6 +346,9 @@
/* Define to 1 if you have the `z' library (-lz). */
#undef HAVE_LIBZ
+/* Define to 1 if you have the `lz4' library (-llz4). */
+#undef HAVE_LIBLZ4
+
/* Define to 1 if you have the `link' function. */
#undef HAVE_LINK
diff --git a/src/include/postgres.h b/src/include/postgres.h
index 2ed572004d..e98af05b30 100644
--- a/src/include/postgres.h
+++ b/src/include/postgres.h
@@ -145,7 +145,8 @@ typedef union
struct /* Compressed-in-line format */
{
uint32 va_header;
- uint32 va_rawsize; /* Original data size (excludes header) */
+ uint32 va_tcinfo; /* Original data size (excludes header) and
+ * compression method */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
} varattrib_4b;
@@ -274,14 +275,23 @@ typedef struct
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
+#define VARHDRSZ_COMPRESS offsetof(varattrib_4b, va_compressed.va_data)
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
+#define VARLENA_RAWSIZE_BITS 30
+#define VARLENA_RAWSIZE_MASK ((1U << VARLENA_RAWSIZE_BITS) - 1)
+
+/*
+ * va_tcinfo in va_compress contains raw size of datum and compression method.
+ */
#define VARRAWSIZE_4B_C(PTR) \
- (((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
+ (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_RAWSIZE_MASK)
+#define VARCOMPRESS_4B_C(PTR) \
+ (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_RAWSIZE_BITS)
/* Externally visible macros */
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
new file mode 100644
index 0000000000..6981d580dc
--- /dev/null
+++ b/src/test/regress/expected/compression.out
@@ -0,0 +1,267 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+ Table "public.cmdata"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+Indexes:
+ "idx" btree (f1)
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+\d+ cmdata1
+ Table "public.cmdata1"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | lz4 | |
+
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+ERROR: column data type integer does not support compression
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+(1 row)
+
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+ substr
+--------
+ 01234
+(1 row)
+
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ substr
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+(1 row)
+
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove3;
+ pg_column_compression
+-----------------------
+ pglz
+ pglz
+(2 rows)
+
+--test composite type
+CREATE TABLE cmdata2 (x cmdata1 compression pglz);
+INSERT INTO cmdata2 SELECT cmdata1 FROM cmdata1;
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+CREATE TABLE cmdata3 (x cmdata2 compression pglz);
+INSERT INTO cmdata3 SELECT row(cmdata1)::cmdata2 FROM cmdata1;
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+DROP TABLE cmdata3;
+DROP TABLE cmdata2;
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+INSERT INTO cmdata1 SELECT * FROM cmdata2;
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+DROP TABLE cmdata2;
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | lz4 | |
+
+DROP TABLE cmdata2;
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | character varying | | | | plain | pglz | |
+
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+\d+ mv
+ Materialized view "public.mv"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ x | text | | | | extended | pglz | |
+View definition:
+ SELECT cmdata1.f1 AS x
+ FROM cmdata1;
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+SELECT pg_column_compression(x) FROM mv;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+SELECT pg_column_compression(f1) FROM cmpart;
+ pg_column_compression
+-----------------------
+ lz4
+ pglz
+(2 rows)
+
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+NOTICE: merging multiple inherited definitions of column "f1"
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+NOTICE: merging column "f1" with inherited definition
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmdata1;
+ length
+--------
+ 10040
+ 12449
+(2 rows)
+
+SELECT length(f1) FROM cmmove1;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove2;
+ length
+--------
+ 10040
+(1 row)
+
+SELECT length(f1) FROM cmmove3;
+ length
+--------
+ 10000
+ 10040
+(2 rows)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
new file mode 100644
index 0000000000..893c18c7fa
--- /dev/null
+++ b/src/test/regress/expected/compression_1.out
@@ -0,0 +1,269 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+ Table "public.cmdata"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+Indexes:
+ "idx" btree (f1)
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+ERROR: unsupported LZ4 compression method
+DETAIL: This functionality requires the server to be built with lz4 support.
+HINT: You need to rebuild PostgreSQL using --with-lz4.
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+ ^
+\d+ cmdata1
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+ERROR: column data type integer does not support compression
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+ substr
+--------
+ 01234
+(1 row)
+
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ ^
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+ ^
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmmove3 SELECT * FROM cmdata1;
+ ^
+SELECT pg_column_compression(f1) FROM cmmove3;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+--test composite type
+CREATE TABLE cmdata2 (x cmdata1 compression pglz);
+ERROR: type "cmdata1" does not exist
+LINE 1: CREATE TABLE cmdata2 (x cmdata1 compression pglz);
+ ^
+INSERT INTO cmdata2 SELECT cmdata1 FROM cmdata1;
+ERROR: relation "cmdata2" does not exist
+LINE 1: INSERT INTO cmdata2 SELECT cmdata1 FROM cmdata1;
+ ^
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+ERROR: relation "cmdata2" does not exist
+LINE 1: SELECT pg_column_compression((x).f1) FROM cmdata2;
+ ^
+CREATE TABLE cmdata3 (x cmdata2 compression pglz);
+ERROR: type "cmdata2" does not exist
+LINE 1: CREATE TABLE cmdata3 (x cmdata2 compression pglz);
+ ^
+INSERT INTO cmdata3 SELECT row(cmdata1)::cmdata2 FROM cmdata1;
+ERROR: relation "cmdata3" does not exist
+LINE 1: INSERT INTO cmdata3 SELECT row(cmdata1)::cmdata2 FROM cmdata...
+ ^
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+ERROR: relation "cmdata2" does not exist
+LINE 1: SELECT pg_column_compression((x).f1) FROM cmdata2;
+ ^
+DROP TABLE cmdata3;
+ERROR: table "cmdata3" does not exist
+DROP TABLE cmdata2;
+ERROR: table "cmdata2" does not exist
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+INSERT INTO cmdata1 SELECT * FROM cmdata2;
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmdata1 SELECT * FROM cmdata2;
+ ^
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+DROP TABLE cmdata2;
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+ERROR: relation "cmdata1" does not exist
+LINE 1: CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+ ^
+\d+ cmdata2
+DROP TABLE cmdata2;
+ERROR: table "cmdata2" does not exist
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | character varying | | | | plain | pglz | |
+
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+ ^
+\d+ mv
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+SELECT pg_column_compression(x) FROM mv;
+ERROR: relation "mv" does not exist
+LINE 1: SELECT pg_column_compression(x) FROM mv;
+ ^
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+ERROR: unsupported LZ4 compression method
+DETAIL: This functionality requires the server to be built with lz4 support.
+HINT: You need to rebuild PostgreSQL using --with-lz4.
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+ERROR: relation "cmpart" does not exist
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+ERROR: relation "cmpart" does not exist
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+ERROR: relation "cmpart" does not exist
+LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',1004));
+ ^
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+ERROR: relation "cmpart" does not exist
+LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',4004));
+ ^
+SELECT pg_column_compression(f1) FROM cmpart;
+ERROR: relation "cmpart" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmpart;
+ ^
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+ERROR: relation "cmdata1" does not exist
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+NOTICE: merging column "f1" with inherited definition
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT length(f1) FROM cmdata1;
+ ^
+SELECT length(f1) FROM cmmove1;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove2;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove3;
+ length
+--------
+ 10000
+(1 row)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index e280198b17..70c38309d7 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -115,7 +115,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# ----------
# Another group of parallel tests
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression
# event triggers cannot run concurrently with any test that runs DDL
# oidjoins is read-only, though, and should run late for best coverage
diff --git a/src/test/regress/pg_regress_main.c b/src/test/regress/pg_regress_main.c
index 8dc4941c24..1524676f3b 100644
--- a/src/test/regress/pg_regress_main.c
+++ b/src/test/regress/pg_regress_main.c
@@ -78,11 +78,11 @@ psql_start_test(const char *testname,
* against different AMs without unnecessary differences.
*/
offset += snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset,
- "\"%s%spsql\" -X -a -q -d \"%s\" -v %s < \"%s\" > \"%s\" 2>&1",
+ "\"%s%spsql\" -X -a -q -d \"%s\" %s < \"%s\" > \"%s\" 2>&1",
bindir ? bindir : "",
bindir ? "/" : "",
dblist->str,
- "HIDE_TABLEAM=\"on\"",
+ "-v HIDE_TABLEAM=on -v HIDE_TOAST_COMPRESSION=on",
infile,
outfile);
if (offset >= sizeof(psql_cmd))
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index 6a57e889a1..d81d04136c 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -201,6 +201,7 @@ test: partition_aggregate
test: partition_info
test: tuplesort
test: explain
+test: compression
test: event_trigger
test: oidjoins
test: fast_default
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
new file mode 100644
index 0000000000..b085c9fbd9
--- /dev/null
+++ b/src/test/regress/sql/compression.sql
@@ -0,0 +1,113 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+\d+ cmdata1
+
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmdata1;
+
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove2;
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove3;
+
+--test composite type
+CREATE TABLE cmdata2 (x cmdata1 compression pglz);
+INSERT INTO cmdata2 SELECT cmdata1 FROM cmdata1;
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+
+CREATE TABLE cmdata3 (x cmdata2 compression pglz);
+INSERT INTO cmdata3 SELECT row(cmdata1)::cmdata2 FROM cmdata1;
+SELECT pg_column_compression((x).f1) FROM cmdata2;
+DROP TABLE cmdata3;
+DROP TABLE cmdata2;
+
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+INSERT INTO cmdata1 SELECT * FROM cmdata2;
+SELECT pg_column_compression(f1) FROM cmdata1;
+DROP TABLE cmdata2;
+
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+\d+ cmdata2
+DROP TABLE cmdata2;
+
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+\d+ mv
+SELECT pg_column_compression(f1) FROM cmdata1;
+SELECT pg_column_compression(x) FROM mv;
+
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+SELECT pg_column_compression(f1) FROM cmpart;
+
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+SELECT length(f1) FROM cmdata1;
+SELECT length(f1) FROM cmmove1;
+SELECT length(f1) FROM cmmove2;
+SELECT length(f1) FROM cmmove3;
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..5f39a92111 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -307,6 +307,7 @@ sub GenerateFiles
HAVE_LIBXML2 => undef,
HAVE_LIBXSLT => undef,
HAVE_LIBZ => $self->{options}->{zlib} ? 1 : undef,
+ HAVE_LIBLZ4 => undef,
HAVE_LINK => undef,
HAVE_LOCALE_T => 1,
HAVE_LONG_INT_64 => undef,
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 574a8a94fa..e6cb35a16f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -395,6 +395,7 @@ CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
CompressionAlgorithm
+CompressionRoutine
CompressorState
ComputeXidHorizonsResult
ConditionVariable
--
2.17.0
--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-f-3nd-attempt-to-use-pkgconfig-to-allow-compiling-on.patch"
^ permalink raw reply [nested|flat] 2+ messages in thread
* UniqueKey v2
@ 2023-10-16 03:09 [email protected]
0 siblings, 0 replies; 2+ messages in thread
From: [email protected] @ 2023-10-16 03:09 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>; David Rowley <[email protected]>; Tom Lane <[email protected]>; Ashutosh Bapat <[email protected]>
Hi,
David and I had worked the uniquekey stuff since 2020[1], and later it
is blocked by the NULL values stuff. Now the blocker should be removed
by Var.varnullingrels, so it is time to work on this again. During the
past 3 years, we have found more and more interesting usage of it.
Here is a design document and a part of implementation.
What is UniqueKey?
-----------------
UniqueKey represents a uniqueness information for a RelOptInfo. for
example:
SELECT id FROM t;
where the ID is the UniqueKey for the RelOptInfo (t). In the real word,
it has the following attributes:
1). It should be EquivalenceClass based. for example:
SELECT a FROM t WHERE a = id;
In this case, the UniqueKey should be 1 EC with two members
- EC(EM(a), EM(id)).
2). Each UniqueKey may be made up with 1+ EquivalenceClass. for example:
CREATE TABLE t(a int not null, b int not null);
CREATE UNIQUE INDEX on t(a, b);
SELECT * FROM t;
Where the UniqueKey for RelOptInfo (t) will be 2 ECs with each 1 has 1
member.
- EC(em=a), EC(em=b)
3). Each RelOptInfo may have 1+ UniqueKeys.
CREATE TABLE t(a int not null, b int not null, c int not null);
CREATE UNIQUE INDEX on t(a, b);
CREATE UNIQUE INDEX on t(c);
SELECT * FROM t;
Where the UniqueKey for RelOptInfo (t) will be
- [EC(em=a), EC(em=b)].
- [EC(em=c)]
4). A special case is about the one-row case. It works like:
SELECT * FROM t WHERE id = 1;
Here every single expression in the RelOptInfo (t) is unique.
Where can we use it?
--------------------
1. mark the distinct as no-op. SELECT DISTINCT uniquekey FROM v; This
optimization has been required several times in our threads.
2. Figure out more pathkey within the onerow case, then some planning
time can be reduced to be big extend. This user case is not absurd, I
run into a real user case like this:
CREATE TABLE small_t (id int primary key, b int, c int .. u int);
CREATE INDEX ON small_t(b);
CREATE INDEX ON small_t(c);
..
SELECT * FROM small_t s
JOIN t1 on t1.sb = s.b
JOIN T2 on t2.sc = s.c
..
JOIN t20 on t20.su = s.u
WHERE s.id = 1;
Without the above optimization, we don't know s.b /s.c is ordered
already, so it might keep more different paths for small_t because of
they have different interesting pathkey, and use more planning time
for sorting to support merge join.
With the above optimization, the planning time should be reduced since
the seq scan can produce a ordered result for every expression.
3. Figure out more interesting pathkey after join with normal UniqueKey.
CREATE TABLE t(id int primary key, b int, c int);
CREATE INDEX on t(c);
ANALYZE t;
explain (costs off)
select t1.id, t2.c from t t1
join t1 t2 on t1.id = t2.b
and t2.c > 3
order by t1.id, t2.c;
QUERY PLAN
--------------------------------------------------
Sort Key: t1.id, t2.c <--- this sort can be avoided actually.
-> Nested Loop
Join Filter: (t1.id = t2.b)
-> Index Only Scan using t_pkey on t t1
-> Index Scan using t1_c_idx on t1 t2
Index Cond: (c > 3)
*Without knowing the t1.id is unique*, which means there are some
duplicated data in t1.id, the duplication data in t1 will break the
order of (t1.id, t2.c), but if we know the t1.id is unique, the sort
will be not needed. I'm pretty happy with this finding.
4. Optimize some group by case, like
SELECT id, sum(b) FROM t GROUP BY id
is same with
SELECT id, b from t;
I'm not sure how often it is in the real life, I'm not so excited with
this for now.
How to present ECs in UniqueKey?
--------------------------------
I choose "Bitmapset *eclass_indexes;" finally, which is because
Bitmapset is memory compact and good at bms_union, bms_is_subset
stuffs. The value in the bitmap is the positions in root->eq_classes. It
is also be able to present the UniqueKey which is made up from multi
relations or upper relation. I'm pleased with the EC strategy because
the existing logic would even create a EC with single members which
means we don't need to create any EquivalenceClass for our own. for
example, in the case of
SELECT DISTINCT pk FROM t;
a EquivalenceClass with single member is created.
How to present single row in UniqueKey
-------------------------------------
I just use a 'Index relid', an non-zero value means the
RelOptInfo[relid] is single row. For the case like
SELECT * FROM t WHERE id = 1;
The UniqueKey is:
- UniqueKey(eclass_indexes=NULL, relid=1)
during a join, any unique keys join with single row, it's uniqueness can
be kept.
SELECT t1.uk, t2.a FROM t WHERE t2.id = 1 and any-qual(t1, t2);
- UniqueKey (t1.uk)
more specially, join two single row like:
SELECT * FROM t1 join t2 on true where t1.id = 1 and t2.id = 2;
the UniqueKey for the JoinRel will be:
- UniqueKey(eclass_indexes=NULL, relid=1)
- UniqueKey(eclass_indexes=NULL, relid=2)
However, the current single row presentation can't works well with Upper
relation, which I think it would be acceptable. See the following case:
SELECT count(*) FROM t1 JOIN t2 on true;
How to maintain the uniquekey?
-------------------------------
the uniquekey is maintained from baserel to join rel then to upper
relation. In the base rel, it comes from unique index. From the join
relation, it is maintained with two rules:
- the uniquekey in one side is still unique if it can't be duplicated
after the join. for example:
SELECT t1.pk FROM t1 JOIN t2 ON t1.a = t2.pk;
UniqueKey on t1: t1.pk
UniqueKey on t1 Join t2: t1.pk
- The combined unique key from both sides are unique all the times.
SELECT t1.pk , t2.pk FROM t1 join t2 on true;
UniqueKey on t1 join t2: (t1.pk, t2.pk)
Some other operations like DISTINCT, GROUP BY can produce UniqueKey as well.
NULL values
-----------
I added notnullattrs in RelOptInfo, which present if these attributes may
not be NULL after the baserestrictinfo is executed. not-null-attributes
may be generated by not-null constraint in catalog or baserestrictinfo
(only) filter. However it is possible become NULLs because of outer
join, then Var.varnullingrels is used in this case. see
'var_is_nullable' function call.
To simplify the UniqueKey module, it doesn't care about the null values
during the maintaining, which means it may contains multi NULL values
all the time by design. However whenever a user case care about that,
the user case can get the answer with the above logic, that is what
'mark-distinct-as-noop' does.
How to reduce the overhead
----------------------------------
UniqueKey employs the similar strategy like PathKey, it only maintain
the interesting PathKey. Currently the interesting UniqueKey includes:
1). It is subset of distinct_pathkeys.
2). It is used in mergeable join clauses for unique key deduction (for
the join rel case, rule 1). In this case, it can be discarded quickly if
the join has been done.
To avoid to check if an uniquekey is subset of distinct clause again and
again, I cached the result into UnqiueKey struct during the UniqueKey
creation.
Since our first goal is just used for marking distinct as no-op, so if
there is no distinct clause at all, unique key will be not maintained at
the beginning. so we can have some codes like:
if (root->distinct_pathkeys == NULL)
return;
This fast path is NOT added for now for better code coverage.
What I have now:
----------------
The current patch just maintain the UniqueKey at the baserel level and
used it for mark-distinct-as-noop purpose. including the basic idea of
- How the UniqueKey is defined.
- How to find out the interesting pathkey in the base relation level.
- How to figure out the unique key contains NULL values.
Also the test cases are prepared, see uniquekey.sql.
Some deep issues can only be found during the development, but I still
like to gather more feedback to see if anything is wrong at the first
place. Like what role will the collation play on for UniqueKey.
Any thought?
Thanks.
[1]
https://www.postgresql.org/message-id/CAApHDvrBXjAvH45dEAZFOk-hzOt1mJC7-fxZ2v49mc5njtA7VQ%40mail.gma...
Best Regards
Andy Fan
Attachments:
[text/x-patch] v1-0001-uniquekey-on-base-relation-and-used-it-for-mark-d.patch (25.4K, ../../[email protected]/2-v1-0001-uniquekey-on-base-relation-and-used-it-for-mark-d.patch)
download | inline diff:
From 1248b106a0e3d759896e95e36bafcda65216a965 Mon Sep 17 00:00:00 2001
From: Andy Fan <[email protected]>
Date: Thu, 12 Oct 2023 13:45:51 +0800
Subject: [PATCH v1 1/1] uniquekey on base relation and used it for
mark-distinct-as-op.
---
src/backend/nodes/list.c | 17 ++
src/backend/optimizer/path/Makefile | 3 +-
src/backend/optimizer/path/allpaths.c | 2 +
src/backend/optimizer/path/equivclass.c | 48 +++
src/backend/optimizer/path/uniquekey.c | 387 ++++++++++++++++++++++++
src/backend/optimizer/plan/initsplan.c | 8 +
src/backend/optimizer/plan/planner.c | 33 ++
src/backend/optimizer/util/plancat.c | 10 +
src/backend/optimizer/util/relnode.c | 2 +
src/include/nodes/pathnodes.h | 19 ++
src/include/nodes/pg_list.h | 2 +
src/include/optimizer/paths.h | 10 +
src/test/regress/expected/uniquekey.out | 69 +++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/uniquekey.sql | 33 ++
15 files changed, 643 insertions(+), 2 deletions(-)
create mode 100644 src/backend/optimizer/path/uniquekey.c
create mode 100644 src/test/regress/expected/uniquekey.out
create mode 100644 src/test/regress/sql/uniquekey.sql
diff --git a/src/backend/nodes/list.c b/src/backend/nodes/list.c
index 750ee5a7e55..eaeaa19ad2d 100644
--- a/src/backend/nodes/list.c
+++ b/src/backend/nodes/list.c
@@ -694,6 +694,23 @@ list_member_ptr(const List *list, const void *datum)
return false;
}
+/*
+ * Return index of the datum in list if found. otherwise return -1.
+ */
+int
+list_member_ptr_pos(const List *list, const void *datum)
+{
+ ListCell *lc;
+
+ foreach(lc, list)
+ {
+ if (lfirst(lc) == datum)
+ return foreach_current_index(lc);
+ }
+
+ return -1;
+}
+
/*
* Return true iff the integer 'datum' is a member of the list.
*/
diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile
index 1e199ff66f7..63cc1505d96 100644
--- a/src/backend/optimizer/path/Makefile
+++ b/src/backend/optimizer/path/Makefile
@@ -21,6 +21,7 @@ OBJS = \
joinpath.o \
joinrels.o \
pathkeys.o \
- tidpath.o
+ tidpath.o \
+ uniquekey.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 3cda88e3333..815a74a9dfa 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -458,6 +458,8 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel,
}
}
+ populate_baserel_uniquekeys(root, rel);
+
/*
* We insist that all non-dummy rels have a nonzero rowcount estimate.
*/
diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c
index 7fa502d6e25..29edc8d1b5a 100644
--- a/src/backend/optimizer/path/equivclass.c
+++ b/src/backend/optimizer/path/equivclass.c
@@ -744,6 +744,54 @@ get_eclass_for_sort_expr(PlannerInfo *root,
return newec;
}
+/*
+ * find_ec_position_matching_expr
+ * Locate the position of EquivalenceClass whose members matching
+ * the given expr, if any; return -1 if no match.
+ */
+int
+find_ec_position_matching_expr(PlannerInfo *root,
+ Expr *expr,
+ RelOptInfo *baserel)
+{
+ int i = -1;
+
+ while ((i = bms_next_member(baserel->eclass_indexes, i)) >= 0)
+ {
+ EquivalenceClass *ec = list_nth(root->eq_classes, i);
+
+ if (find_ec_member_matching_expr(ec, expr, baserel->relids))
+ return i;
+ }
+ return -1;
+}
+
+/*
+ * build_ec_positions_for_exprs
+ *
+ * Given a list of exprs, find the related EC positions for each of
+ * them. if any exprs has no EC related, return NULL;
+ */
+Bitmapset *
+build_ec_positions_for_exprs(PlannerInfo *root, List *exprs, RelOptInfo *rel)
+{
+ ListCell *lc;
+ Bitmapset *res = NULL;
+
+ foreach(lc, exprs)
+ {
+ int pos = find_ec_position_matching_expr(root, lfirst(lc), rel);
+
+ if (pos < 0)
+ {
+ bms_free(res);
+ return NULL;
+ }
+ res = bms_add_member(res, pos);
+ }
+ return res;
+}
+
/*
* find_ec_member_matching_expr
* Locate an EquivalenceClass member matching the given expr, if any;
diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c
new file mode 100644
index 00000000000..c23379083e3
--- /dev/null
+++ b/src/backend/optimizer/path/uniquekey.c
@@ -0,0 +1,387 @@
+/*-------------------------------------------------------------------------
+ *
+ * uniquekey.c
+ * Utilities for maintaining uniquekey.
+ *
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/optimizer/path/uniquekey.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/sysattr.h"
+#include "nodes/nodeFuncs.h"
+#include "nodes/pathnodes.h"
+#include "optimizer/optimizer.h"
+#include "optimizer/paths.h"
+
+
+/* Functions to populate UniqueKey */
+static bool add_uniquekey_for_uniqueindex(PlannerInfo *root,
+ IndexOptInfo *unique_index,
+ List *truncatable_exprs,
+ List *expr_opfamilies);
+static bool unique_ecs_useful_for_distinct(PlannerInfo *root, Bitmapset *ec_indexes);
+
+/* Helper functions to create UniqueKey. */
+static UniqueKey * make_uniquekey(Bitmapset *eclass_indexes,
+ bool useful_for_distinct);
+static void mark_rel_singlerow(RelOptInfo *rel, int relid);
+
+static UniqueKey * rel_singlerow_uniquekey(RelOptInfo *rel);
+static bool uniquekey_contains_multinulls(PlannerInfo *root, RelOptInfo *rel, UniqueKey * ukey);
+
+/* Debug only */
+static void print_uniquekey(PlannerInfo *root, RelOptInfo *rel);
+
+
+/*
+ * populate_baserel_uniquekeys
+ *
+ * UniqueKey on baserel comes from unique indexes. Any expression
+ * which equals with Const can be stripped and the left expression are
+ * still unique.
+ */
+void
+populate_baserel_uniquekeys(PlannerInfo *root, RelOptInfo *rel)
+{
+ ListCell *lc;
+ List *truncatable_exprs = NIL,
+ *expr_opfamilies = NIL;
+
+ foreach(lc, rel->baserestrictinfo)
+ {
+ RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
+
+ if (rinfo->mergeopfamilies == NIL)
+ continue;
+
+ if (!IsA(rinfo->clause, OpExpr))
+ continue;
+
+ if (bms_is_empty(rinfo->left_relids))
+ truncatable_exprs = lappend(truncatable_exprs, get_rightop(rinfo->clause));
+ else if (bms_is_empty(rinfo->right_relids))
+ truncatable_exprs = lappend(truncatable_exprs, get_leftop(rinfo->clause));
+ else
+ continue;
+
+ expr_opfamilies = lappend(expr_opfamilies, rinfo->mergeopfamilies);
+ }
+
+ foreach(lc, rel->indexlist)
+ {
+ IndexOptInfo *index = (IndexOptInfo *) lfirst(lc);
+
+ if (!index->unique || !index->immediate ||
+ (index->indpred != NIL && !index->predOK))
+ continue;
+
+ if (add_uniquekey_for_uniqueindex(root, index,
+ truncatable_exprs,
+ expr_opfamilies))
+ /* Find a singlerow case, no need to go through other indexes. */
+ return;
+ }
+
+ print_uniquekey(root, rel);
+}
+
+
+/*
+ * add_uniquekey_for_uniqueindex
+ *
+ * populate a UniqueKey if it is interesting, return true iff the
+ * UniqueKey is an SingleRow. Only the interesting UniqueKeys are kept.
+ */
+static bool
+add_uniquekey_for_uniqueindex(PlannerInfo *root, IndexOptInfo *unique_index,
+ List *truncatable_exprs, List *expr_opfamilies)
+{
+ List *unique_exprs = NIL;
+ Bitmapset *unique_ecs = NULL;
+ ListCell *indexpr_item;
+ RelOptInfo *rel = unique_index->rel;
+ bool used_for_distinct;
+ int c;
+
+ indexpr_item = list_head(unique_index->indexprs);
+
+ for (c = 0; c < unique_index->nkeycolumns; c++)
+ {
+ int attr = unique_index->indexkeys[c];
+ Expr *expr; /* The candidate for UniqueKey expression. */
+ bool matched_const = false;
+ ListCell *lc1,
+ *lc2;
+
+ if (attr > 0)
+ {
+ Var *var;
+
+ expr = list_nth_node(TargetEntry, unique_index->indextlist, c)->expr;
+ var = castNode(Var, expr);
+ }
+ else if (attr == 0)
+ {
+ /* Expression index */
+ expr = lfirst(indexpr_item);
+ indexpr_item = lnext(unique_index->indexprs, indexpr_item);
+ }
+ else /* attr < 0 */
+ {
+ /* Index on OID is possible, not handle it for now. */
+ return false;
+ }
+
+ /* Ignore the expr which are equals to const. */
+ forboth(lc1, truncatable_exprs, lc2, expr_opfamilies)
+ {
+ if (list_member_oid((List *) lfirst(lc2), unique_index->opfamily[c]) &&
+ match_index_to_operand((Node *) lfirst(lc1), c, unique_index))
+ {
+ matched_const = true;
+ break;
+ }
+ }
+
+ if (matched_const)
+ continue;
+
+ unique_exprs = lappend(unique_exprs, expr);
+ }
+
+ if (unique_exprs == NIL)
+ {
+ /* single row is always interesting. */
+ mark_rel_singlerow(rel, rel->relid);
+ return true;
+ }
+
+ /*
+ * if no EquivalenceClass is found for any exprs in unique exprs, we are
+ * sure the whole exprs are not in the DISTINCT clause or mergeable join
+ * clauses. so it is not interesting.
+ */
+ unique_ecs = build_ec_positions_for_exprs(root, unique_exprs, rel);
+ if (unique_ecs == NULL)
+ return false;
+
+ used_for_distinct = unique_ecs_useful_for_distinct(root, unique_ecs);
+
+
+ rel->uniquekeys = lappend(rel->uniquekeys,
+ make_uniquekey(unique_ecs,
+ used_for_distinct));
+ return false;
+}
+
+/*
+ * make_uniquekey
+ * Based on UnqiueKey rules, it is impossible for a UnqiueKey
+ * which have eclass_indexes and relid both set. This function just
+ * handle eclass_indexes case.
+ */
+static UniqueKey *
+make_uniquekey(Bitmapset *eclass_indexes, bool useful_for_distinct)
+{
+ UniqueKey *ukey = makeNode(UniqueKey);
+
+ ukey->eclass_indexes = eclass_indexes;
+ ukey->relid = 0;
+ ukey->use_for_distinct = useful_for_distinct;
+ return ukey;
+}
+
+/*
+ * mark_rel_singlerow
+ * mark a relation as singlerow.
+ */
+static void
+mark_rel_singlerow(RelOptInfo *rel, int relid)
+{
+ UniqueKey *ukey = makeNode(UniqueKey);
+
+ ukey->relid = relid;
+ rel->uniquekeys = list_make1(ukey);
+}
+
+/*
+ *
+ * Return the UniqueKey if rel is a singlerow Relation. othwise
+ * return NULL.
+ */
+static UniqueKey *
+rel_singlerow_uniquekey(RelOptInfo *rel)
+{
+ if (rel->uniquekeys != NIL)
+ {
+ UniqueKey *ukey = linitial_node(UniqueKey, rel->uniquekeys);
+
+ if (ukey->relid)
+ return ukey;
+ }
+ return NULL;
+}
+
+/*
+ * print_uniquekey
+ * Used for easier reivew, should be removed before commit.
+ */
+static void
+print_uniquekey(PlannerInfo *root, RelOptInfo *rel)
+{
+ if (!enable_geqo)
+ {
+ ListCell *lc;
+
+ elog(INFO, "Rel = %s", bmsToString(rel->relids));
+ foreach(lc, rel->uniquekeys)
+ {
+ UniqueKey *ukey = lfirst_node(UniqueKey, lc);
+
+ elog(INFO, "UNIQUEKEY{indexes=%s, singlerow_rels=%d, use_for_distinct=%d}",
+ bmsToString(ukey->eclass_indexes),
+ ukey->relid,
+ ukey->use_for_distinct);
+ }
+ }
+}
+
+/*
+ * is it possible that the var contains multi NULL values in the given
+ * RelOptInfo rel?
+ */
+static bool
+var_is_nullable(PlannerInfo *root, Var *var, RelOptInfo *rel)
+{
+ RelOptInfo *base_rel;
+
+ /* check if the outer join can add the NULL values. */
+ if (bms_overlap(var->varnullingrels, rel->relids))
+ return true;
+
+ /* check if the user data has the NULL values. */
+ base_rel = root->simple_rel_array[var->varno];
+ return !bms_is_member(var->varattno - FirstLowInvalidHeapAttributeNumber, base_rel->notnullattrs);
+}
+
+
+/*
+ * uniquekey_contains_multinulls
+ *
+ * Check if the uniquekey contains nulls values.
+ */
+static bool
+uniquekey_contains_multinulls(PlannerInfo *root, RelOptInfo *rel, UniqueKey * ukey)
+{
+ int i = -1;
+
+ while ((i = bms_next_member(rel->eclass_indexes, i)) >= 0)
+ {
+ EquivalenceClass *ec = list_nth_node(EquivalenceClass, root->eq_classes, i);
+ ListCell *lc;
+
+ foreach(lc, ec->ec_members)
+ {
+ EquivalenceMember *em = lfirst_node(EquivalenceMember, lc);
+ Var *var;
+
+ var = (Var *) em->em_expr;
+
+ if (!IsA(var, Var))
+ continue;
+
+ if (var_is_nullable(root, var, rel))
+ return true;
+ else
+
+ /*
+ * If any one of member in the EC is not nullable, we all the
+ * members are not nullable since they are equal with each
+ * other.
+ */
+ break;
+ }
+ }
+
+ return false;
+}
+
+
+/*
+ * relation_is_distinct_for
+ *
+ * Check if the rel is distinct for distinct_pathkey.
+ */
+bool
+relation_is_distinct_for(PlannerInfo *root, RelOptInfo *rel, List *distinct_pathkey)
+{
+ ListCell *lc;
+ UniqueKey *singlerow_ukey = rel_singlerow_uniquekey(rel);
+ Bitmapset *pathkey_bm = NULL;
+
+ if (singlerow_ukey)
+ {
+ return !uniquekey_contains_multinulls(root, rel, singlerow_ukey);
+ }
+
+ foreach(lc, distinct_pathkey)
+ {
+ PathKey *pathkey = lfirst_node(PathKey, lc);
+ int pos = list_member_ptr_pos(root->eq_classes, pathkey->pk_eclass);
+
+ if (pos == -1)
+ return false;
+
+ pathkey_bm = bms_add_member(pathkey_bm, pos);
+ }
+
+ foreach(lc, rel->uniquekeys)
+ {
+ UniqueKey *ukey = lfirst_node(UniqueKey, lc);
+
+ if (bms_is_subset(ukey->eclass_indexes, pathkey_bm) &&
+ !uniquekey_contains_multinulls(root, rel, ukey))
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * unique_ecs_useful_for_distinct
+ *
+ * Return true if all the EquivalenceClass for ecs exists in root->distinct_pathkey.
+ */
+static bool
+unique_ecs_useful_for_distinct(PlannerInfo *root, Bitmapset *ec_indexes)
+{
+ int i = -1;
+
+ while ((i = bms_next_member(ec_indexes, i)) >= 0)
+ {
+ EquivalenceClass *ec = list_nth(root->eq_classes, i);
+ ListCell *p;
+ bool found = false;
+
+ foreach(p, root->distinct_pathkeys)
+ {
+ PathKey *pathkey = lfirst_node(PathKey, p);
+
+ if (ec == pathkey->pk_eclass)
+ {
+ found = true;
+ break;
+ }
+ }
+ if (!found)
+ return false;
+ }
+ return true;
+}
diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c
index b31d8921211..1ed02d6eb2e 100644
--- a/src/backend/optimizer/plan/initsplan.c
+++ b/src/backend/optimizer/plan/initsplan.c
@@ -2647,6 +2647,14 @@ distribute_restrictinfo_to_rels(PlannerInfo *root,
/* Add clause to rel's restriction list */
rel->baserestrictinfo = lappend(rel->baserestrictinfo,
restrictinfo);
+ {
+ List *not_null_vars = find_nonnullable_vars((Node *) restrictinfo->clause);
+
+ if (not_null_vars != NIL)
+ rel->notnullattrs = bms_union(rel->notnullattrs,
+ list_nth(not_null_vars, rel->relid));
+ }
+
/* Update security level info */
rel->baserestrict_min_security = Min(rel->baserestrict_min_security,
restrictinfo->security_level);
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index aafef24cf14..31ae33e5062 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -1663,9 +1663,12 @@ grouping_planner(PlannerInfo *root, double tuple_fraction)
gset_data);
/* Fix things up if grouping_target contains SRFs */
if (parse->hasTargetSRFs)
+ {
adjust_paths_for_srfs(root, current_rel,
grouping_targets,
grouping_targets_contain_srfs);
+ current_rel->uniquekeys = NIL;
+ }
}
/*
@@ -1725,6 +1728,7 @@ grouping_planner(PlannerInfo *root, double tuple_fraction)
* Now we are prepared to build the final-output upperrel.
*/
final_rel = fetch_upper_rel(root, UPPERREL_FINAL, NULL);
+ /* simple_copy_uniquekeys(final_rel, current_rel); */
/*
* If the input rel is marked consider_parallel and there's nothing that's
@@ -4049,6 +4053,22 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel,
gd,
extra->targetList);
+ if (root->parse->groupingSets)
+ {
+ /* nothing to do */
+ }
+ else if (root->parse->groupClause && root->group_pathkeys != NIL)
+ {
+ /*
+ * populate_uniquekeys_from_pathkeys(root, grouped_rel,
+ * root->group_pathkeys);
+ */
+ }
+ else
+ {
+ /* SingleRow Case */
+ }
+
/* Build final grouping paths */
add_paths_to_grouping_rel(root, input_rel, grouped_rel,
partially_grouped_rel, agg_costs, gd,
@@ -4668,9 +4688,22 @@ create_distinct_paths(PlannerInfo *root, RelOptInfo *input_rel)
{
RelOptInfo *distinct_rel;
+ /*
+ * distinct_pathkeys may be NIL if it distinctClause is not sortable. XXX:
+ * What should we do for the else?
+ */
+ if (root->distinct_pathkeys &&
+ relation_is_distinct_for(root, input_rel, root->distinct_pathkeys))
+ return input_rel;
+
/* For now, do all work in the (DISTINCT, NULL) upperrel */
distinct_rel = fetch_upper_rel(root, UPPERREL_DISTINCT, NULL);
+ /*
+ * populate_uniquekeys_from_pathkeys(root, distinct_rel,
+ * root->distinct_pathkeys);
+ */
+
/*
* We don't compute anything at this level, so distinct_rel will be
* parallel-safe if the input rel is parallel-safe. In particular, if
diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c
index 7159c775fbd..665d3e890cf 100644
--- a/src/backend/optimizer/util/plancat.c
+++ b/src/backend/optimizer/util/plancat.c
@@ -121,6 +121,7 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent,
Relation relation;
bool hasindex;
List *indexinfos = NIL;
+ Index i;
/*
* We need not lock the relation since it was already locked, either by
@@ -175,6 +176,15 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent,
/* Retrieve the parallel_workers reloption, or -1 if not set. */
rel->rel_parallel_workers = RelationGetParallelWorkers(relation, -1);
+ for (i = 0; i < relation->rd_att->natts; i++)
+ {
+ FormData_pg_attribute attr = relation->rd_att->attrs[i];
+
+ if (attr.attnotnull)
+ rel->notnullattrs = bms_add_member(rel->notnullattrs,
+ attr.attnum - FirstLowInvalidHeapAttributeNumber);
+ }
+
/*
* Make list of indexes. Ignore indexes on system catalogs if told to.
* Don't bother with indexes from traditional inheritance parents. For
diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index 5d83f60eb9a..2c9e1fe0164 100644
--- a/src/backend/optimizer/util/relnode.c
+++ b/src/backend/optimizer/util/relnode.c
@@ -284,6 +284,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent)
rel->all_partrels = NULL;
rel->partexprs = NULL;
rel->nullable_partexprs = NULL;
+ rel->uniquekeys = NIL;
/*
* Pass assorted information down the inheritance hierarchy.
@@ -745,6 +746,7 @@ build_join_rel(PlannerInfo *root,
joinrel->all_partrels = NULL;
joinrel->partexprs = NULL;
joinrel->nullable_partexprs = NULL;
+ joinrel->uniquekeys = NIL;
/* Compute information relevant to the foreign relations. */
set_foreign_rel_properties(joinrel, outer_rel, inner_rel);
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index 5702fbba60c..0b9e3fe1293 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -876,6 +876,7 @@ typedef struct RelOptInfo
* Vars/Exprs, cost, width
*/
struct PathTarget *reltarget;
+ List *uniquekeys; /* A list of UniqueKey. */
/*
* materialization information
@@ -913,6 +914,9 @@ typedef struct RelOptInfo
Relids *attr_needed pg_node_attr(read_write_ignore);
/* array indexed [min_attr .. max_attr] */
int32 *attr_widths pg_node_attr(read_write_ignore);
+
+ /* The not null attrs from catalogs or baserestrictinfo. */
+ Bitmapset *notnullattrs;
/* relids of outer joins that can null this baserel */
Relids nulling_relids;
/* LATERAL Vars and PHVs referenced by rel */
@@ -1456,6 +1460,21 @@ typedef struct PathKey
bool pk_nulls_first; /* do NULLs come before normal values? */
} PathKey;
+
+typedef struct UniqueKey
+{
+ pg_node_attr(no_read, no_query_jumble)
+
+ NodeTag type;
+ Bitmapset *eclass_indexes; /* Indexes in PlannerInfo's eq_class list of
+ * ECs that is unique for a RelOptInfo. */
+ int relid;
+ bool use_for_distinct; /* true if it is used in distinct-pathkey,
+ * in this case we would never check if we
+ * should discard it during join search. */
+} UniqueKey;
+
+
/*
* VolatileFunctionStatus -- allows nodes to cache their
* contain_volatile_functions properties. VOLATILITY_UNKNOWN means not yet
diff --git a/src/include/nodes/pg_list.h b/src/include/nodes/pg_list.h
index 529a382d284..1be53fe7f76 100644
--- a/src/include/nodes/pg_list.h
+++ b/src/include/nodes/pg_list.h
@@ -581,6 +581,8 @@ extern bool list_member_int(const List *list, int datum);
extern bool list_member_oid(const List *list, Oid datum);
extern bool list_member_xid(const List *list, TransactionId datum);
+extern int list_member_ptr_pos(const List *list, const void *datum);
+
extern pg_nodiscard List *list_delete(List *list, void *datum);
extern pg_nodiscard List *list_delete_ptr(List *list, void *datum);
extern pg_nodiscard List *list_delete_int(List *list, int datum);
diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h
index 7b896d821e8..3ac25d47ae7 100644
--- a/src/include/optimizer/paths.h
+++ b/src/include/optimizer/paths.h
@@ -136,11 +136,17 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root,
extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec,
Expr *expr,
Relids relids);
+extern int find_ec_position_matching_expr(PlannerInfo *root,
+ Expr *expr,
+ RelOptInfo *rel);
extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root,
EquivalenceClass *ec,
List *exprs,
Relids relids,
bool require_parallel_safe);
+extern Bitmapset *build_ec_positions_for_exprs(PlannerInfo *root,
+ List *exprs,
+ RelOptInfo *rel);
extern bool relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel,
EquivalenceClass *ec,
bool require_parallel_safe);
@@ -259,4 +265,8 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root,
extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel,
List *live_childrels);
+extern bool relation_is_distinct_for(PlannerInfo *root, RelOptInfo *rel,
+ List *distinct_pathkey);
+extern void populate_baserel_uniquekeys(PlannerInfo *root,
+ RelOptInfo *baserel);
#endif /* PATHS_H */
diff --git a/src/test/regress/expected/uniquekey.out b/src/test/regress/expected/uniquekey.out
new file mode 100644
index 00000000000..43b876edd1a
--- /dev/null
+++ b/src/test/regress/expected/uniquekey.out
@@ -0,0 +1,69 @@
+CREATE TABLE uk_t (id int primary key, a int not null, b int not null, c int, d int, e int);
+explain (costs off)
+select distinct id from uk_t;
+ QUERY PLAN
+------------------
+ Seq Scan on uk_t
+(1 row)
+
+explain (costs off)
+select distinct e from uk_t where id = e;
+ QUERY PLAN
+--------------------
+ Seq Scan on uk_t
+ Filter: (id = e)
+(2 rows)
+
+create unique index on uk_t (a, b);
+create unique index on uk_t (c, d);
+explain (costs off)
+select distinct a, b from uk_t;
+ QUERY PLAN
+------------------
+ Seq Scan on uk_t
+(1 row)
+
+explain (costs off)
+select distinct c, d from uk_t;
+ QUERY PLAN
+------------------------
+ HashAggregate
+ Group Key: c, d
+ -> Seq Scan on uk_t
+(3 rows)
+
+explain (costs off)
+select distinct c, d from uk_t
+where c > 0 and d > 0;
+ QUERY PLAN
+-------------------------------------------
+ Bitmap Heap Scan on uk_t
+ Recheck Cond: ((c > 0) AND (d > 0))
+ -> Bitmap Index Scan on uk_t_c_d_idx
+ Index Cond: ((c > 0) AND (d > 0))
+(4 rows)
+
+explain (costs off)
+select distinct d from uk_t
+where c > 1 and d > 0;
+ QUERY PLAN
+-------------------------------------------------
+ HashAggregate
+ Group Key: d
+ -> Bitmap Heap Scan on uk_t
+ Recheck Cond: ((c > 1) AND (d > 0))
+ -> Bitmap Index Scan on uk_t_c_d_idx
+ Index Cond: ((c > 1) AND (d > 0))
+(6 rows)
+
+explain (costs off)
+select distinct d from uk_t
+where c = 1 and d > 0;
+ QUERY PLAN
+-------------------------------------------
+ Bitmap Heap Scan on uk_t
+ Recheck Cond: ((c = 1) AND (d > 0))
+ -> Bitmap Index Scan on uk_t_c_d_idx
+ Index Cond: ((c = 1) AND (d > 0))
+(4 rows)
+
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 4df9d8503b9..a6531a1f3af 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -62,7 +62,7 @@ test: sanity_check
# join depends on create_misc
# ----------
test: select_into select_distinct select_distinct_on select_implicit select_having subselect union case join aggregates transactions random portals arrays btree_index hash_index update delete namespace prepared_xacts
-
+test: uniquekey
# ----------
# Another group of parallel tests
# ----------
diff --git a/src/test/regress/sql/uniquekey.sql b/src/test/regress/sql/uniquekey.sql
new file mode 100644
index 00000000000..a18cd81b3fe
--- /dev/null
+++ b/src/test/regress/sql/uniquekey.sql
@@ -0,0 +1,33 @@
+CREATE TABLE uk_t (id int primary key, a int not null, b int not null, c int, d int, e int);
+
+explain (costs off)
+select distinct id from uk_t;
+
+explain (costs off)
+select distinct e from uk_t where id = e;
+
+create unique index on uk_t (a, b);
+create unique index on uk_t (c, d);
+
+explain (costs off)
+select distinct a, b from uk_t;
+
+explain (costs off)
+select distinct c, d from uk_t;
+
+explain (costs off)
+select distinct c, d from uk_t
+where c > 0 and d > 0;
+
+explain (costs off)
+select distinct d from uk_t
+where c > 1 and d > 0;
+
+explain (costs off)
+select distinct d from uk_t
+where c = 1 and d > 0;
+
+
+
+
+
--
2.21.0
^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2023-10-16 03:09 UTC | newest]
Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-05 04:03 [PATCH 4/8] Built-in compression method Dilip Kumar <[email protected]>
2023-10-16 03:09 UniqueKey v2 [email protected]
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox