Received: from malur.postgresql.org ([217.196.149.56]) by arkaria.postgresql.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_CBC_SHA384:256) (Exim 4.89) (envelope-from ) id 1f0W5K-0000Ut-4h for pgsql-hackers@arkaria.postgresql.org; Mon, 26 Mar 2018 17:39:07 +0000 Received: from localhost ([127.0.0.1] helo=malur.postgresql.org) by malur.postgresql.org with esmtp (Exim 4.89) (envelope-from ) id 1f0W5I-0004QI-Rt for pgsql-hackers@arkaria.postgresql.org; Mon, 26 Mar 2018 17:39:04 +0000 Received: from magus.postgresql.org ([2a02:c0:301:0:ffff::29]) by malur.postgresql.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_CBC_SHA384:256) (Exim 4.89) (envelope-from ) id 1f0W50-0002OW-Li for pgsql-hackers@lists.postgresql.org; Mon, 26 Mar 2018 17:38:47 +0000 Received: from mail.postgrespro.ru ([93.174.131.138]) by magus.postgresql.org with esmtp (Exim 4.89) (envelope-from ) id 1f0W4l-00037T-5W for pgsql-hackers@postgresql.org; Mon, 26 Mar 2018 17:38:45 +0000 Received: from localhost (localhost [127.0.0.1]) by mail.postgrespro.ru (Postfix) with ESMTP id 204AC21C1FB6; Mon, 26 Mar 2018 20:38:30 +0300 (MSK) X-Virus-Scanned: Debian amavisd-new at postgrespro.ru X-Spam-Flag: NO X-Spam-Score: 0 X-Spam-Level: X-Spam-Status: No, score=x tagged_above=-99 required=4 WHITELISTED tests=[] autolearn=unavailable Received: from localhost.localdomain (broadband-46-242-9-190.moscow.rt.ru [46.242.9.190]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mail.postgrespro.ru (Postfix) with ESMTPSA id 3A34C21C1FB5; Mon, 26 Mar 2018 20:38:29 +0300 (MSK) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=postgrespro.ru; s=mail; t=1522085909; bh=mvHv7TB3JRB8K/8vhFQOIbEK94l0rEIDVcQ2envQ8/o=; h=Date:From:To:Cc:Subject:In-Reply-To:References; b=HaXxAbwuT2Q0ovJSpeUmC9kIr2VXmmLItaJPqwZ0vOp3zay3IVkepwy73AGVMtYMo mOn+G6LHtvEcldnGdfvTm2VXjzdWjr07b1UYtoq5mXOqsxyXShwV85JqFv9j2r4zQw nxeggOjxoMyffRz3TiFcjEcBZzj3bdMKsz0O2nhA= Date: Mon, 26 Mar 2018 20:38:25 +0300 From: Ildus Kurbangaliev To: Ildar Musin Cc: Robert Haas , Alexander Korotkov , Tomas Vondra , Alvaro Herrera , =?UTF-8?B?0JXQstCz0LXQvdC40Lkg?= =?UTF-8?B?0KjQuNGI0LrQuNC9?= , Andres Freund , Oleg Bartunov , Craig Ringer , Peter Eisentraut , PostgreSQL Hackers , Chapman Flack Subject: Re: [HACKERS] Custom compression methods Message-ID: <20180326203825.142eedaa@postgrespro.ru> In-Reply-To: <20180314163123.583b2998@wp.localdomain> References: <20171201194859.le5hvnnrjzhxhm2t@alvherre.pgsql> <20171213151818.75a20259@postgrespro.ru> <20171218115431.2b3e29ba@wp.localdomain> <20180115024930.48583c69@hh> <0633c6e5-5328-5c5b-708b-c4409e3fd131@postgrespro.ru> <20180123160454.131ade09@wp.localdomain> <6fd5dea3-8889-c7bb-9df2-79493b2b6ab0@postgrespro.ru> <20180125172457.5967c022@wp.localdomain> <20180129144452.08f04dc3@wp.localdomain> <20180206134731.4ba75fb8@wp.localdomain> <20180226152556.0c6b000f@wp.localdomain> <20180314163123.583b2998@wp.localdomain> X-Mailer: Claws Mail 3.16.0 (GTK+ 2.24.32; x86_64-pc-linux-gnu) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="MP_/awZ+sOzvK+HW9/=nLSA94yJ" List-Id: List-Help: List-Subscribe: List-Post: List-Owner: List-Archive: Precedence: bulk --MP_/awZ+sOzvK+HW9/=nLSA94yJ Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: 7bit Content-Disposition: inline Attached rebased version of the patch. Fixed conflicts in pg_class.h. -- ---- Regards, Ildus Kurbangaliev --MP_/awZ+sOzvK+HW9/=nLSA94yJ Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=custom_compression_methods_v14.patch diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out index b7c76469fc..71251cb8cb 100644 --- a/contrib/test_decoding/expected/ddl.out +++ b/contrib/test_decoding/expected/ddl.out @@ -420,12 +420,12 @@ CREATE TABLE replication_metadata ( WITH (user_catalog_table = true) ; \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=true @@ -434,12 +434,12 @@ INSERT INTO replication_metadata(relation, options) VALUES ('foo', ARRAY['a', 'b']); ALTER TABLE replication_metadata RESET (user_catalog_table); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) @@ -447,12 +447,12 @@ INSERT INTO replication_metadata(relation, options) VALUES ('bar', ARRAY['a', 'b']); ALTER TABLE replication_metadata SET (user_catalog_table = true); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=true @@ -465,13 +465,13 @@ ALTER TABLE replication_metadata ALTER COLUMN rewritemeornot TYPE text; ERROR: cannot rewrite table "replication_metadata" used as a catalog table ALTER TABLE replication_metadata SET (user_catalog_table = false); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | - rewritemeornot | integer | | | | plain | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | + rewritemeornot | integer | | | | plain | | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=false diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 95a5b113b9..8118b92e0a 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -57,7 +57,7 @@ pg_am - index access methods + access methods @@ -70,6 +70,11 @@ access method support procedures + + pg_attr_compression + table columns compression relationships and options + + pg_attrdef column default values @@ -587,8 +592,10 @@ The catalog pg_am stores information about relation access methods. There is one row for each access method supported by the system. - Currently, only indexes have access methods. The requirements for index - access methods are discussed in detail in . + Currently, compression and index access methods are supported. + The requirements for index access methods are discussed in detail + in , for compression access methods + could be found in . @@ -892,6 +899,12 @@ + + <structname>pg_attr_compression</structname> + + pg_attr_compression + + <structname>pg_attrdef</structname> diff --git a/doc/src/sgml/compression-am.sgml b/doc/src/sgml/compression-am.sgml new file mode 100644 index 0000000000..8950c5d1ec --- /dev/null +++ b/doc/src/sgml/compression-am.sgml @@ -0,0 +1,145 @@ + + + + Compression Access Method Interface Definition + + This chapter defines the interface between the core + PostgreSQL system and compression access + methods, which manage individual compression types. + + + + Basic API Structure for compression methods + + + Each compression access method is described by a row in the + pg_am + system catalog. The pg_am entry + specifies a name and a handler function for the access + method. These entries can be created and deleted using the + and + SQL commands. + + + + A compression access method handler function must be declared to accept a + single argument of type internal and to return the + pseudo-type compression_am_handler. The argument is a dummy value that + simply serves to prevent handler functions from being called directly from + SQL commands. The result of the function must be a palloc'd struct of + type CompressionAmRoutine, which contains everything + that the core code needs to know to make use of the compression access method. + The CompressionAmRoutine struct, also called the access + method's API struct, contains pointers to support + functions for the access method. These support functions are plain C + functions and are not visible or callable at the SQL level. + The support functions are described in . + + + + The structure CompressionAmRoutine is defined thus: + +typedef struct CompressionAmRoutine +{ + NodeTag type; + + cmcheck_function cmcheck; /* can be NULL */ + cmdrop_function cmdrop; /* can be NULL */ + cminitstate_function cminitstate; /* can be NULL */ + cmcompress_function cmcompress; + cmcompress_function cmdecompress; +} CompressionAmRoutine; + + + + + Compression Access Method Functions + + + The compression and auxiliary functions that an compression access + method must provide in CompressionAmRoutine are: + + + + +void +cmcheck (Form_pg_attribute att, List *options); + + Called when an attribute is linked with compression access method. Could + be used to check compatibility with the attribute and other additional + checks. + + + + +void +cmdrop (Oid acoid); + + Called when the attribute compression is going to be removed. + + + + Compression functions take special struct + CompressionAmOptions as first + parameter. This struct contains per backend cached state for each + attribute compression record. CompressionAmOptions is defined thus: + + +typedef struct CompressionAmOptions +{ + Oid acoid; /* Oid of attribute compression */ + Oid amoid; /* Oid of compression access method */ + List *acoptions; /* Parsed options, used for comparison */ + CompressionAmRoutine *amroutine; /* compression access method routine */ + + /* result of cminitstate function will be put here */ + void *acstate; +} CompressionAmOptions; + + + + + The acstate field is used to keep temporary state + between compression functions calls and stores the result of + cminitstate function. It could be useful to store + the parsed view of the compression options. + + + + Note that any pg_attr_compression relation invalidation + will cause all the cached acstate options cleared. + They will be recreated on the next compression functions calls. + + + + +void * +cminitstate (Oid acoid, List *options); + + Called when CompressionAmOptions is being + initialized. Can return a pointer to memory that will be passed between + compression functions calls. + + + + +struct varlena * +cmcompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to compress varlena. Could return NULL if data is + incompressible. If it returns varlena bigger than original the core will + not use it. + + + + +struct varlena * +cmdecompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to decompress varlena. + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 732b8ab7d0..59716f31aa 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -90,6 +90,7 @@ + diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index a7f6c8dc6a..586c4e30c8 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -47,7 +47,7 @@ Basic API Structure for Indexes - Each index access method is described by a row in the + Each index access method is described by a row with INDEX type in the pg_am system catalog. The pg_am entry specifies a name and a handler function for the access diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 054347b17d..229990941d 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -251,6 +251,7 @@ &custom-scan; &geqo; &indexam; + &compression-am; &generic-wal; &btree; &gist; diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index afe213910c..9a0b2902cd 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -53,6 +53,7 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER [ COLUMN ] column_name SET COMPRESSION compression_am [ WITH (compression_am_options) ] [ PRESERVE (compression_preserve_list) ] ADD table_constraint [ NOT VALID ] ADD table_constraint_using_index ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -351,6 +352,23 @@ WITH ( MODULUS numeric_literal, REM + + + SET COMPRESSION compression_method_name [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) ] + + + + This form adds compression to a column. Compression access method should be + created with . If compression + method has options they could be specified with WITH + parameter. The PRESERVE list contains list of compression access methods + used on the column and determines which of them should be kept on the + column. Without PRESERVE or partial list of compression methods table + will be rewritten. + + + + ADD table_constraint [ NOT VALID ] diff --git a/doc/src/sgml/ref/create_access_method.sgml b/doc/src/sgml/ref/create_access_method.sgml index 851c5e63be..a35005aca3 100644 --- a/doc/src/sgml/ref/create_access_method.sgml +++ b/doc/src/sgml/ref/create_access_method.sgml @@ -61,7 +61,7 @@ CREATE ACCESS METHOD name This clause specifies the type of access method to define. - Only INDEX is supported at present. + INDEX and COMPRESSION types are supported at present. @@ -79,6 +79,9 @@ CREATE ACCESS METHOD name be index_am_handler. The C-level API that the handler function must implement varies depending on the type of access method. The index access method API is described in . + For COMPRESSION access methods, the type must be + compression_am_handler. The compression access method API + is described in . diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 14a43b45e9..0c0a50b6aa 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -65,6 +65,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | + COMPRESSION compression_access_method [ WITH (compression_am_options) ] | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE action ] [ ON UPDATE action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -834,6 +835,18 @@ WITH ( MODULUS numeric_literal, REM + + COMPRESSION compression_access_method [ WITH (compression_am_options) ] + + + This clause adds compression to a column. Compression method could be + created with . If compression + method has options they could be specified by WITH + parameter. + + + + EXCLUDE [ USING index_method ] ( exclude_element WITH operator [, ... ] ) index_parameters [ WHERE ( predicate ) ] diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index c0e548fa5b..59c381c572 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -385,10 +385,12 @@ Further details appear in . -The compression technique used for either in-line or out-of-line compressed +The default compression technique used for either in-line or out-of-line compressed data is a fairly simple and very fast member of the LZ family of compression techniques. See -src/common/pg_lzcompress.c for the details. +src/common/pg_lzcompress.c for the details. Also custom +compressions could be used. Look at for +more information. diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index bd93a6a8d1..4cd4fd9715 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -8,7 +8,7 @@ subdir = src/backend/access top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = brin common gin gist hash heap index nbtree rmgrdesc spgist \ - tablesample transam +SUBDIRS = brin common compression gin gist hash heap index nbtree \ + rmgrdesc spgist tablesample transam include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 00316b899c..cb9024863d 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -96,6 +96,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, int keyno; int idxattno; uint16 phony_infomask = 0; + uint16 phony_infomask2 = 0; bits8 *phony_nullbitmap; Size len, hoff, @@ -187,6 +188,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, (char *) rettuple + hoff, data_len, &phony_infomask, + &phony_infomask2, phony_nullbitmap); /* done with these */ diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index c45a48812b..c697a68b18 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -145,7 +145,7 @@ void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, char *data, Size data_size, - uint16 *infomask, bits8 *bit) + uint16 *infomask, uint16 *infomask2, bits8 *bit) { bits8 *bitP; int bitmask; @@ -169,6 +169,7 @@ heap_fill_tuple(TupleDesc tupleDesc, } *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTERNAL); + *infomask2 &= ~HEAP_HASCUSTOMCOMPRESSED; for (i = 0; i < numberOfAttributes; i++) { @@ -195,6 +196,9 @@ heap_fill_tuple(TupleDesc tupleDesc, *bitP |= bitmask; } + if (OidIsValid(att->attcompression)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; + /* * XXX we use the att_align macros on the pointer value itself, not on * an offset. This is a bit of a hack. @@ -213,6 +217,7 @@ heap_fill_tuple(TupleDesc tupleDesc, Pointer val = DatumGetPointer(values[i]); *infomask |= HEAP_HASVARWIDTH; + if (VARATT_IS_EXTERNAL(val)) { if (VARATT_IS_EXTERNAL_EXPANDED(val)) @@ -230,10 +235,20 @@ heap_fill_tuple(TupleDesc tupleDesc, } else { + *infomask |= HEAP_HASEXTERNAL; /* no alignment, since it's short by definition */ data_length = VARSIZE_EXTERNAL(val); memcpy(data, val, data_length); + + if (VARATT_IS_EXTERNAL_ONDISK(val)) + { + struct varatt_external toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, val); + if (VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; + } } } else if (VARATT_IS_SHORT(val)) @@ -257,6 +272,9 @@ heap_fill_tuple(TupleDesc tupleDesc, att->attalign); data_length = VARSIZE(val); memcpy(data, val, data_length); + + if (VARATT_IS_CUSTOM_COMPRESSED(val)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; } } else if (att->attlen == -2) @@ -774,6 +792,7 @@ heap_form_tuple(TupleDesc tupleDescriptor, (char *) td + hoff, data_len, &td->t_infomask, + &td->t_infomask2, (hasnull ? td->t_bits : NULL)); return tuple; @@ -1482,6 +1501,7 @@ heap_form_minimal_tuple(TupleDesc tupleDescriptor, (char *) tuple + hoff, data_len, &tuple->t_infomask, + &tuple->t_infomask2, (hasnull ? tuple->t_bits : NULL)); return tuple; diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index f7103e53bc..d1417445ab 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -47,6 +47,7 @@ index_form_tuple(TupleDesc tupleDescriptor, unsigned short infomask = 0; bool hasnull = false; uint16 tupmask = 0; + uint16 tupmask2 = 0; int numberOfAttributes = tupleDescriptor->natts; #ifdef TOAST_INDEX_HACK @@ -74,13 +75,30 @@ index_form_tuple(TupleDesc tupleDescriptor, /* * If value is stored EXTERNAL, must fetch it so we are not depending - * on outside storage. This should be improved someday. + * on outside storage. This should be improved someday. If value also + * was compressed by custom compression method then we should + * decompress it too. */ if (VARATT_IS_EXTERNAL(DatumGetPointer(values[i]))) + { + struct varatt_external toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, DatumGetPointer(values[i])); + if (VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer)) + untoasted_values[i] = + PointerGetDatum(heap_tuple_untoast_attr((struct varlena *) + DatumGetPointer(values[i]))); + else + untoasted_values[i] = + PointerGetDatum(heap_tuple_fetch_attr((struct varlena *) + DatumGetPointer(values[i]))); + untoasted_free[i] = true; + } + else if (VARATT_IS_CUSTOM_COMPRESSED(DatumGetPointer(values[i]))) { untoasted_values[i] = - PointerGetDatum(heap_tuple_fetch_attr((struct varlena *) - DatumGetPointer(values[i]))); + PointerGetDatum(heap_tuple_untoast_attr((struct varlena *) + DatumGetPointer(values[i]))); untoasted_free[i] = true; } @@ -92,7 +110,7 @@ index_form_tuple(TupleDesc tupleDescriptor, VARSIZE(DatumGetPointer(untoasted_values[i])) > TOAST_INDEX_TARGET && (att->attstorage == 'x' || att->attstorage == 'm')) { - Datum cvalue = toast_compress_datum(untoasted_values[i]); + Datum cvalue = toast_compress_datum(untoasted_values[i], InvalidOid); if (DatumGetPointer(cvalue) != NULL) { @@ -142,6 +160,7 @@ index_form_tuple(TupleDesc tupleDescriptor, (char *) tp + hoff, data_size, &tupmask, + &tupmask2, (hasnull ? (bits8 *) tp + sizeof(IndexTupleData) : NULL)); #ifdef TOAST_INDEX_HACK diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 46276ceff1..16e17444ce 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -944,11 +944,100 @@ untransformRelOptions(Datum options) val = (Node *) makeString(pstrdup(p)); } result = lappend(result, makeDefElem(pstrdup(s), val, -1)); + pfree(s); } return result; } +/* + * helper function for qsort to compare DefElem + */ +static int +compare_options(const void *a, const void *b) +{ + DefElem *da = (DefElem *) lfirst(*(ListCell **) a); + DefElem *db = (DefElem *) lfirst(*(ListCell **) b); + + return strcmp(da->defname, db->defname); +} + +/* + * Convert a DefElem list to the text array format that is used in + * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, + * pg_foreign_table and pg_attr_compression + * + * Returns the array in the form of a Datum, or PointerGetDatum(NULL) + * if the list is empty. + * + * Note: The array is usually stored to database without further + * processing, hence any validation should be done before this + * conversion. + */ +Datum +optionListToArray(List *options, bool sorted) +{ + ArrayBuildState *astate = NULL; + ListCell *cell; + List *resoptions = NIL; + int len = list_length(options); + + /* sort by option name if needed */ + if (sorted && len > 1) + resoptions = list_qsort(options, compare_options); + else + resoptions = options; + + foreach(cell, resoptions) + { + DefElem *def = lfirst(cell); + const char *value; + Size len; + text *t; + + value = defGetString(def); + len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); + t = palloc(len + 1); + SET_VARSIZE(t, len); + sprintf(VARDATA(t), "%s=%s", def->defname, value); + + astate = accumArrayResult(astate, PointerGetDatum(t), + false, TEXTOID, + CurrentMemoryContext); + } + + /* free if the the modified version of list was used */ + if (resoptions != options) + list_free(resoptions); + + if (astate) + return makeArrayResult(astate, CurrentMemoryContext); + + return PointerGetDatum(NULL); +} + +/* + * Return human readable list of reloptions + */ +char * +formatRelOptions(List *options) +{ + StringInfoData buf; + ListCell *cell; + + initStringInfo(&buf); + + foreach(cell, options) + { + DefElem *def = (DefElem *) lfirst(cell); + + appendStringInfo(&buf, "%s%s=%s", buf.len > 0 ? ", " : "", + def->defname, defGetString(def)); + } + + return buf.data; +} + /* * Extract and parse reloptions from a pg_class tuple. * diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index f1f44230cd..e43818a5e4 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -21,6 +21,7 @@ #include "access/hash.h" #include "access/htup_details.h" +#include "access/reloptions.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "miscadmin.h" @@ -72,6 +73,7 @@ CreateTemplateTupleDesc(int natts, bool hasoid) desc->tdtypeid = RECORDOID; desc->tdtypmod = -1; desc->tdhasoid = hasoid; + desc->tdflags = 0; desc->tdrefcount = -1; /* assume not reference-counted */ return desc; @@ -94,8 +96,17 @@ CreateTupleDesc(int natts, bool hasoid, Form_pg_attribute *attrs) desc = CreateTemplateTupleDesc(natts, hasoid); for (i = 0; i < natts; ++i) + { memcpy(TupleDescAttr(desc, i), attrs[i], ATTRIBUTE_FIXED_PART_SIZE); + /* + * If even one of attributes is compressed we save information about + * it to TupleDesc flags + */ + if (OidIsValid(attrs[i]->attcompression)) + desc->tdflags |= TD_ATTR_CUSTOM_COMPRESSED; + } + return desc; } @@ -135,6 +146,7 @@ CreateTupleDescCopy(TupleDesc tupdesc) /* We can copy the tuple type identification, too */ desc->tdtypeid = tupdesc->tdtypeid; desc->tdtypmod = tupdesc->tdtypmod; + desc->tdflags = tupdesc->tdflags; return desc; } @@ -197,6 +209,7 @@ CreateTupleDescCopyConstr(TupleDesc tupdesc) /* We can copy the tuple type identification, too */ desc->tdtypeid = tupdesc->tdtypeid; desc->tdtypmod = tupdesc->tdtypmod; + desc->tdflags = tupdesc->tdflags; return desc; } @@ -280,6 +293,7 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, dstAtt->attnotnull = false; dstAtt->atthasdef = false; dstAtt->attidentity = '\0'; + dstAtt->attcompression = InvalidOid; } /* @@ -384,6 +398,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (tupdesc1->tdhasoid != tupdesc2->tdhasoid) return false; + if (tupdesc1->tdflags != tupdesc2->tdflags) + return false; for (i = 0; i < tupdesc1->natts; i++) { @@ -434,6 +450,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (attr1->attcollation != attr2->attcollation) return false; + if (attr1->attcompression != attr2->attcompression) + return false; /* attacl, attoptions and attfdwoptions are not even present... */ } @@ -496,6 +514,7 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) } else if (tupdesc2->constr != NULL) return false; + return true; } @@ -601,6 +620,7 @@ TupleDescInitEntry(TupleDesc desc, att->attalign = typeForm->typalign; att->attstorage = typeForm->typstorage; att->attcollation = typeForm->typcollation; + att->attcompression = InvalidOid; ReleaseSysCache(tuple); } @@ -713,7 +733,6 @@ TupleDescInitEntryCollation(TupleDesc desc, TupleDescAttr(desc, attributeNumber - 1)->attcollation = collationid; } - /* * BuildDescForRelation * diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile new file mode 100644 index 0000000000..14286920d3 --- /dev/null +++ b/src/backend/access/compression/Makefile @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/compression +# +# IDENTIFICATION +# src/backend/access/compression/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/compression +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = cm_pglz.o cmapi.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..76463b84ef --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,168 @@ +/*------------------------------------------------------------------------- + * + * pglz_cm.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cmdrop = NULL; /* no drop behavior */ + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/backend/access/compression/cmapi.c b/src/backend/access/compression/cmapi.c new file mode 100644 index 0000000000..0ebf6e46a8 --- /dev/null +++ b/src/backend/access/compression/cmapi.c @@ -0,0 +1,123 @@ +/*------------------------------------------------------------------------- + * + * compression/cmapi.c + * Functions for compression access methods + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/compression/cmapi.c + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "fmgr.h" +#include "access/cmapi.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "catalog/pg_am.h" +#include "catalog/pg_attr_compression.h" +#include "commands/defrem.h" +#include "utils/syscache.h" + +/* + * InvokeCompressionAmHandler - call the specified access method handler routine to get + * its CompressionAmRoutine struct, which will be palloc'd in the caller's context. + */ +CompressionAmRoutine * +InvokeCompressionAmHandler(Oid amhandler) +{ + Datum datum; + CompressionAmRoutine *routine; + + datum = OidFunctionCall0(amhandler); + routine = (CompressionAmRoutine *) DatumGetPointer(datum); + + if (routine == NULL || !IsA(routine, CompressionAmRoutine)) + elog(ERROR, "compression method handler function %u " + "did not return an CompressionAmRoutine struct", + amhandler); + + return routine; +} + +/* + * GetCompressionAmRoutine + * + * Return CompressionAmRoutine by attribute compression Oid + */ +CompressionAmRoutine * +GetCompressionAmRoutine(Oid acoid) +{ + Oid amoid; + regproc amhandler; + HeapTuple tuple; + Form_pg_attr_compression acform; + + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + /* extract access method name and get handler for it */ + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_compression_am_oid(NameStr(acform->acname), false); + amhandler = get_am_handler_oid(amoid, AMTYPE_COMPRESSION, false); + ReleaseSysCache(tuple); + + /* And finally, call the handler function to get the API struct. */ + return InvokeCompressionAmHandler(amhandler); +} + +/* + * GetAttrCompressionOptions + * + * Parse array of attribute compression options and return it as a list. + */ +List * +GetAttrCompressionOptions(Oid acoid) +{ + HeapTuple tuple; + List *result = NIL; + bool isnull; + Datum acoptions; + + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for an attribute compression %u", acoid); + + /* options could be NULL, so we can't use form struct */ + acoptions = SysCacheGetAttr(ATTCOMPRESSIONOID, tuple, + Anum_pg_attr_compression_acoptions, &isnull); + + if (!isnull) + result = untransformRelOptions(acoptions); + + ReleaseSysCache(tuple); + return result; +} + +/* + * GetAttrCompressionAmOid + * + * Return access method Oid by attribute compression Oid + */ +Oid +GetAttrCompressionAmOid(Oid acoid) +{ + Oid result; + HeapTuple tuple; + Form_pg_attr_compression acform; + + /* extract access method Oid */ + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + result = get_compression_am_oid(NameStr(acform->acname), false); + ReleaseSysCache(tuple); + + return result; +} diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index c08ab14c02..75eaf0ae08 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -93,7 +93,8 @@ static HeapScanDesc heap_beginscan_internal(Relation relation, static void heap_parallelscan_startblock_init(HeapScanDesc scan); static BlockNumber heap_parallelscan_nextpage(HeapScanDesc scan); static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, - TransactionId xid, CommandId cid, int options); + TransactionId xid, CommandId cid, int options, + BulkInsertState bistate); static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, Buffer newbuf, HeapTuple oldtup, HeapTuple newtup, HeapTuple old_key_tup, @@ -2356,13 +2357,14 @@ UpdateXmaxHintBits(HeapTupleHeader tuple, Buffer buffer, TransactionId xid) * GetBulkInsertState - prepare status object for a bulk insert */ BulkInsertState -GetBulkInsertState(void) +GetBulkInsertState(HTAB *preserved_am_info) { BulkInsertState bistate; bistate = (BulkInsertState) palloc(sizeof(BulkInsertStateData)); bistate->strategy = GetAccessStrategy(BAS_BULKWRITE); bistate->current_buf = InvalidBuffer; + bistate->preserved_am_info = preserved_am_info; return bistate; } @@ -2449,7 +2451,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, * Note: below this point, heaptup is the data we actually intend to store * into the relation; tup is the caller's original untoasted data. */ - heaptup = heap_prepare_insert(relation, tup, xid, cid, options); + heaptup = heap_prepare_insert(relation, tup, xid, cid, options, bistate); /* * Find buffer to insert this tuple into. If the page is all visible, @@ -2618,7 +2620,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, - CommandId cid, int options) + CommandId cid, int options, BulkInsertState bistate) { /* * Parallel operations are required to be strictly read-only in a parallel @@ -2679,8 +2681,12 @@ heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, Assert(!HeapTupleHasExternal(tup)); return tup; } - else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD) - return toast_insert_or_update(relation, tup, NULL, options); + else if (HeapTupleHasExternal(tup) + || RelationGetDescr(relation)->tdflags & TD_ATTR_CUSTOM_COMPRESSED + || HeapTupleHasCustomCompressed(tup) + || tup->t_len > TOAST_TUPLE_THRESHOLD) + return toast_insert_or_update(relation, tup, NULL, options, + bistate ? bistate->preserved_am_info : NULL); else return tup; } @@ -2719,7 +2725,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, heaptuples = palloc(ntuples * sizeof(HeapTuple)); for (i = 0; i < ntuples; i++) heaptuples[i] = heap_prepare_insert(relation, tuples[i], - xid, cid, options); + xid, cid, options, bistate); /* * Allocate some memory to use for constructing the WAL record. Using @@ -4010,6 +4016,8 @@ l2: else need_toast = (HeapTupleHasExternal(&oldtup) || HeapTupleHasExternal(newtup) || + RelationGetDescr(relation)->tdflags & TD_ATTR_CUSTOM_COMPRESSED || + HeapTupleHasCustomCompressed(newtup) || newtup->t_len > TOAST_TUPLE_THRESHOLD); pagefree = PageGetHeapFreeSpace(page); @@ -4112,7 +4120,7 @@ l2: if (need_toast) { /* Note we always use WAL and FSM during updates */ - heaptup = toast_insert_or_update(relation, newtup, &oldtup, 0); + heaptup = toast_insert_or_update(relation, newtup, &oldtup, 0, NULL); newtupsize = MAXALIGN(heaptup->t_len); } else diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 7d466c2588..4f3c99a70a 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -654,7 +654,8 @@ raw_heap_insert(RewriteState state, HeapTuple tup) heaptup = toast_insert_or_update(state->rs_new_rel, tup, NULL, HEAP_INSERT_SKIP_FSM | (state->rs_use_wal ? - 0 : HEAP_INSERT_SKIP_WAL)); + 0 : HEAP_INSERT_SKIP_WAL), + NULL); else heaptup = tup; diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index 546f80f05c..e845c360b5 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -30,17 +30,24 @@ #include #include +#include "access/cmapi.h" #include "access/genam.h" #include "access/heapam.h" +#include "access/reloptions.h" #include "access/tuptoaster.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "commands/defrem.h" #include "common/pg_lzcompress.h" #include "miscadmin.h" #include "utils/expandeddatum.h" #include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/inval.h" +#include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" +#include "utils/syscache.h" #include "utils/typcache.h" #include "utils/tqual.h" @@ -53,19 +60,51 @@ typedef struct toast_compress_header { int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 rawsize; + uint32 info; /* flags (2 bits) and rawsize */ } toast_compress_header; +/* + * If the compression method were used, then data also contains + * Oid of compression options + */ +typedef struct toast_compress_header_custom +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + uint32 info; /* flags (2 high bits) and rawsize */ + Oid cmid; /* Oid from pg_attr_compression */ +} toast_compress_header_custom; + +static HTAB *amoptions_cache = NULL; +static MemoryContext amoptions_cache_mcxt = NULL; + +#define RAWSIZEMASK (0x3FFFFFFFU) + /* * Utilities for manipulation of header information for compressed * toast entries. + * + * Since version 11 TOAST_COMPRESS_SET_RAWSIZE also marks compressed + * varlenas as custom compressed. Such varlenas will contain 0x02 (0b10) in + * two highest bits. */ -#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) -#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize) +#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) +#define TOAST_COMPRESS_HDRSZ_CUSTOM ((int32) sizeof(toast_compress_header_custom)) +#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->info & RAWSIZEMASK) #define TOAST_COMPRESS_RAWDATA(ptr) \ (((char *) (ptr)) + TOAST_COMPRESS_HDRSZ) #define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \ - (((toast_compress_header *) (ptr))->rawsize = (len)) +do { \ + Assert(len > 0 && len <= RAWSIZEMASK); \ + ((toast_compress_header *) (ptr))->info = (len) | (0x02 << 30); \ +} while (0) +#define TOAST_COMPRESS_SET_CMID(ptr, oid) \ + (((toast_compress_header_custom *) (ptr))->cmid = (oid)) + +#define VARATT_EXTERNAL_SET_CUSTOM(toast_pointer) \ +do { \ + ((toast_pointer).va_extinfo |= (1 << 31)); \ + ((toast_pointer).va_extinfo &= ~(1 << 30)); \ +} while (0) static void toast_delete_datum(Relation rel, Datum value, bool is_speculative); static Datum toast_save_datum(Relation rel, Datum value, @@ -83,6 +122,9 @@ static int toast_open_indexes(Relation toastrel, static void toast_close_indexes(Relation *toastidxs, int num_indexes, LOCKMODE lock); static void init_toast_snapshot(Snapshot toast_snapshot); +static void init_amoptions_cache(void); +static CompressionAmOptions *lookup_amoptions(Oid acoid); +static bool attr_compression_options_are_equal(Oid acoid1, Oid acoid2); /* ---------- @@ -421,7 +463,7 @@ toast_datum_size(Datum value) struct varatt_external toast_pointer; VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_extsize; + result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -511,6 +553,98 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) } } +/* ---------- + * toast_prepare_varlena + * + * Untoast or fetch varlena if needed. This function could return + * compressed varlena. + */ +static struct varlena * +toast_prepare_varlena(Form_pg_attribute att, struct varlena *value, + List *preserved_amoids, bool *modified) +{ + struct varlena *tmpval = NULL; + + *modified = false; + + /* + * We took care of UPDATE, so any external value we find still in the + * tuple must be someone else's that we cannot reuse (this includes the + * case of an out-of-line in-memory datum). Fetch it back (without + * decompression, unless we are forcing PLAIN storage or it has a custom + * compression). If necessary, we'll push it out as a new external value + * below. + */ + if (VARATT_IS_EXTERNAL(value)) + { + *modified = true; + if (att->attstorage == 'p') + { + value = heap_tuple_untoast_attr(value); + + /* untoasted value does not need any further work */ + return value; + } + else + value = heap_tuple_fetch_attr(value); + } + + /* + * Process custom compressed datum. + * + * 1) If destination column has identical compression move the data as is + * and only change attribute compression Oid. 2) If it's rewrite from + * ALTER command check list of preserved compression access methods. 3) In + * other cases just untoast the datum. + */ + if (VARATT_IS_CUSTOM_COMPRESSED(value)) + { + bool storage_ok = (att->attstorage == 'm' || att->attstorage == 'x'); + toast_compress_header_custom *hdr; + + hdr = (toast_compress_header_custom *) value; + + /* nothing to do */ + if (storage_ok && hdr->cmid == att->attcompression) + return value; + + if (storage_ok && + OidIsValid(att->attcompression) && + attr_compression_options_are_equal(att->attcompression, hdr->cmid)) + { + /* identical compression, just change Oid to new one */ + tmpval = palloc(VARSIZE(value)); + memcpy(tmpval, value, VARSIZE(value)); + TOAST_COMPRESS_SET_CMID(tmpval, att->attcompression); + } + else if (preserved_amoids != NULL) + { + Oid amoid = GetAttrCompressionAmOid(hdr->cmid); + + /* decompress the value if it's not in preserved list */ + if (!list_member_oid(preserved_amoids, amoid)) + tmpval = heap_tuple_untoast_attr(value); + } + else + /* just decompress the value */ + tmpval = heap_tuple_untoast_attr(value); + + } + else if (OidIsValid(att->attcompression)) + tmpval = heap_tuple_untoast_attr(value); + + if (tmpval && tmpval != value) + { + /* if value was external we need to free fetched data */ + if (*modified) + pfree(value); + + value = tmpval; + *modified = true; + } + + return value; +} /* ---------- * toast_insert_or_update - @@ -522,6 +656,8 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) * newtup: the candidate new tuple to be inserted * oldtup: the old row version for UPDATE, or NULL for INSERT * options: options to be passed to heap_insert() for toast rows + * preserved_am_info: hash table with attnum as key, for lists of compression + * methods which should be preserved on decompression. * Result: * either newtup if no toasting is needed, or a palloc'd modified tuple * that is what should actually get stored @@ -532,7 +668,7 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) */ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, - int options) + int options, HTAB *preserved_am_info) { HeapTuple result_tuple; TupleDesc tupleDesc; @@ -667,27 +803,37 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ if (att->attlen == -1) { + bool modified = false; + List *preserved_amoids = NIL; + /* * If the table's attribute says PLAIN always, force it so. */ if (att->attstorage == 'p') toast_action[i] = 'p'; + if (VARATT_IS_EXTERNAL(new_value)) + toast_oldexternal[i] = new_value; + /* - * We took care of UPDATE above, so any external value we find - * still in the tuple must be someone else's that we cannot reuse - * (this includes the case of an out-of-line in-memory datum). - * Fetch it back (without decompression, unless we are forcing - * PLAIN storage). If necessary, we'll push it out as a new - * external value below. + * If it's ALTER SET COMPRESSION command we should check that + * access method of compression in preserved list. */ - if (VARATT_IS_EXTERNAL(new_value)) + if (preserved_am_info != NULL) + { + bool found; + AttrCmPreservedInfo *pinfo; + + pinfo = hash_search(preserved_am_info, &att->attnum, + HASH_FIND, &found); + if (found) + preserved_amoids = pinfo->preserved_amoids; + } + + new_value = toast_prepare_varlena(att, + new_value, preserved_amoids, &modified); + if (modified) { - toast_oldexternal[i] = new_value; - if (att->attstorage == 'p') - new_value = heap_tuple_untoast_attr(new_value); - else - new_value = heap_tuple_fetch_attr(new_value); toast_values[i] = PointerGetDatum(new_value); toast_free[i] = true; need_change = true; @@ -741,12 +887,14 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, Datum old_value; Datum new_value; + Form_pg_attribute att; + /* * Search for the biggest yet unprocessed internal attribute */ for (i = 0; i < numAttrs; i++) { - Form_pg_attribute att = TupleDescAttr(tupleDesc, i); + char attstorage; if (toast_action[i] != ' ') continue; @@ -754,7 +902,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; /* can't happen, toast_action would be 'p' */ if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i]))) continue; - if (att->attstorage != 'x' && att->attstorage != 'e') + + attstorage = (TupleDescAttr(tupleDesc, i))->attstorage; + if (attstorage != 'x' && attstorage != 'e') continue; if (toast_sizes[i] > biggest_size) { @@ -770,10 +920,11 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * Attempt to compress it inline, if it has attstorage 'x' */ i = biggest_attno; - if (TupleDescAttr(tupleDesc, i)->attstorage == 'x') + att = TupleDescAttr(tupleDesc, i); + if (att->attstorage == 'x') { old_value = toast_values[i]; - new_value = toast_compress_datum(old_value); + new_value = toast_compress_datum(old_value, att->attcompression); if (DatumGetPointer(new_value) != NULL) { @@ -914,7 +1065,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ i = biggest_attno; old_value = toast_values[i]; - new_value = toast_compress_datum(old_value); + new_value = toast_compress_datum(old_value, + TupleDescAttr(tupleDesc, i)->attcompression); if (DatumGetPointer(new_value) != NULL) { @@ -1046,6 +1198,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, (char *) new_data + new_header_len, new_data_len, &(new_data->t_infomask), + &(new_data->t_infomask2), has_nulls ? new_data->t_bits : NULL); } else @@ -1274,6 +1427,7 @@ toast_flatten_tuple_to_datum(HeapTupleHeader tup, (char *) new_data + new_header_len, new_data_len, &(new_data->t_infomask), + &(new_data->t_infomask2), has_nulls ? new_data->t_bits : NULL); /* @@ -1353,7 +1507,6 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, return new_tuple; } - /* ---------- * toast_compress_datum - * @@ -1368,54 +1521,48 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, * ---------- */ Datum -toast_compress_datum(Datum value) +toast_compress_datum(Datum value, Oid acoid) { - struct varlena *tmp; - int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); - int32 len; + struct varlena *tmp = NULL; + int32 valsize; + CompressionAmOptions *cmoptions = NULL; Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value))); Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value))); - /* - * No point in wasting a palloc cycle if value size is out of the allowed - * range for compression - */ - if (valsize < PGLZ_strategy_default->min_input_size || - valsize > PGLZ_strategy_default->max_input_size) - return PointerGetDatum(NULL); + /* Fallback to default compression if not specified */ + if (!OidIsValid(acoid)) + acoid = DefaultCompressionOid; - tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + - TOAST_COMPRESS_HDRSZ); + cmoptions = lookup_amoptions(acoid); + tmp = cmoptions->amroutine->cmcompress(cmoptions, (const struct varlena *) value); + if (!tmp) + return PointerGetDatum(NULL); /* - * We recheck the actual size even if pglz_compress() reports success, - * because it might be satisfied with having saved as little as one byte - * in the compressed data --- which could turn into a net loss once you - * consider header and alignment padding. Worst case, the compressed - * format might require three padding bytes (plus header, which is - * included in VARSIZE(tmp)), whereas the uncompressed format would take - * only one header byte and no padding if the value is short enough. So - * we insist on a savings of more than 2 bytes to ensure we have a gain. + * We recheck the actual size even if compression function reports + * success, because it might be satisfied with having saved as little as + * one byte in the compressed data --- which could turn into a net loss + * once you consider header and alignment padding. Worst case, the + * compressed format might require three padding bytes (plus header, which + * is included in VARSIZE(tmp)), whereas the uncompressed format would + * take only one header byte and no padding if the value is short enough. + * So we insist on a savings of more than 2 bytes to ensure we have a + * gain. */ - len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), - valsize, - TOAST_COMPRESS_RAWDATA(tmp), - PGLZ_strategy_default); - if (len >= 0 && - len + TOAST_COMPRESS_HDRSZ < valsize - 2) + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + + if (VARSIZE(tmp) < valsize - 2) { TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize); - SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ); + TOAST_COMPRESS_SET_CMID(tmp, cmoptions->acoid); /* successful compression */ return PointerGetDatum(tmp); } - else - { - /* incompressible data */ - pfree(tmp); - return PointerGetDatum(NULL); - } + + /* incompressible data */ + pfree(tmp); + return PointerGetDatum(NULL); } @@ -1510,19 +1657,20 @@ toast_save_datum(Relation rel, Datum value, &num_indexes); /* - * Get the data pointer and length, and compute va_rawsize and va_extsize. + * Get the data pointer and length, and compute va_rawsize and va_extinfo. * * va_rawsize is the size of the equivalent fully uncompressed datum, so * we have to adjust for short headers. * - * va_extsize is the actual size of the data payload in the toast records. + * va_extinfo contains the actual size of the data payload in the toast + * records. */ if (VARATT_IS_SHORT(dval)) { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; /* no flags */ } else if (VARATT_IS_COMPRESSED(dval)) { @@ -1530,7 +1678,10 @@ toast_save_datum(Relation rel, Datum value, data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ toast_pointer.va_rawsize = VARRAWSIZE_4B_C(dval) + VARHDRSZ; - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; + if (VARATT_IS_CUSTOM_COMPRESSED(dval)) + VARATT_EXTERNAL_SET_CUSTOM(toast_pointer); + /* Assert that the numbers look like it's compressed */ Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); } @@ -1539,7 +1690,7 @@ toast_save_datum(Relation rel, Datum value, data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; /* no flags */ } /* @@ -1899,7 +2050,7 @@ toast_fetch_datum(struct varlena *attr) /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - ressize = toast_pointer.va_extsize; + ressize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; result = (struct varlena *) palloc(ressize + VARHDRSZ); @@ -2084,7 +2235,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length) */ Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); - attrsize = toast_pointer.va_extsize; + attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; if (sliceoffset >= attrsize) @@ -2280,15 +2431,26 @@ toast_decompress_datum(struct varlena *attr) Assert(VARATT_IS_COMPRESSED(attr)); - result = (struct varlena *) - palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); - SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + if (VARATT_IS_CUSTOM_COMPRESSED(attr)) + { + CompressionAmOptions *cmoptions; + toast_compress_header_custom *hdr; - if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), - VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, - VARDATA(result), - TOAST_COMPRESS_RAWSIZE(attr)) < 0) - elog(ERROR, "compressed data is corrupted"); + hdr = (toast_compress_header_custom *) attr; + cmoptions = lookup_amoptions(hdr->cmid); + result = cmoptions->amroutine->cmdecompress(cmoptions, attr); + } + else + { + result = (struct varlena *) + palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), + VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, + VARDATA(result), + TOAST_COMPRESS_RAWSIZE(attr)) < 0) + elog(ERROR, "compressed data is corrupted"); + } return result; } @@ -2390,3 +2552,142 @@ init_toast_snapshot(Snapshot toast_snapshot) InitToastSnapshot(*toast_snapshot, snapshot->lsn, snapshot->whenTaken); } + +/* ---------- + * remove_cached_amoptions + * + * Remove specified compression options from cache + */ +static void +remove_cached_amoptions(CompressionAmOptions *entry) +{ + if (entry->amroutine) + pfree(entry->amroutine); + if (entry->acoptions) + list_free_deep(entry->acoptions); + if (entry->acstate) + pfree(entry->acstate); + + if (hash_search(amoptions_cache, + (void *) &entry->acoid, + HASH_REMOVE, + NULL) == NULL) + elog(ERROR, "hash table corrupted"); +} + +/* ---------- + * invalidate_amoptions_cache + * + * Flush cache entries when pg_attr_compression is updated. + */ +static void +invalidate_amoptions_cache(Datum arg, int cacheid, uint32 hashvalue) +{ + HASH_SEQ_STATUS status; + CompressionAmOptions *entry; + + hash_seq_init(&status, amoptions_cache); + while ((entry = (CompressionAmOptions *) hash_seq_search(&status)) != NULL) + remove_cached_amoptions(entry); +} + +/* ---------- + * init_amoptions_cache + * + * Initialize a local cache for attribute compression options. + */ +static void +init_amoptions_cache(void) +{ + HASHCTL ctl; + + amoptions_cache_mcxt = AllocSetContextCreate(TopMemoryContext, + "attr compression cache", + ALLOCSET_DEFAULT_SIZES); + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(CompressionAmOptions); + ctl.hcxt = amoptions_cache_mcxt; + amoptions_cache = hash_create("attr compression cache", 100, &ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + + /* Set up invalidation callback on pg_attr_compression. */ + CacheRegisterSyscacheCallback(ATTCOMPRESSIONOID, + invalidate_amoptions_cache, + (Datum) 0); +} + +/* ---------- + * lookup_amoptions + * + * Return or generate cache record for attribute compression. + */ +static CompressionAmOptions * +lookup_amoptions(Oid acoid) +{ + bool found; + CompressionAmOptions *result; + + if (!amoptions_cache) + init_amoptions_cache(); + + result = hash_search(amoptions_cache, &acoid, HASH_ENTER, &found); + if (!found) + { + MemoryContext oldcxt; + + result->amroutine = NULL; + result->acoptions = NULL; + result->acstate = NULL; + + /* fill access method options in cache */ + oldcxt = MemoryContextSwitchTo(amoptions_cache_mcxt); + + PG_TRY(); + { + result->acoid = acoid; + result->amoid = GetAttrCompressionAmOid(acoid); + result->amroutine = GetCompressionAmRoutine(acoid); + result->acoptions = GetAttrCompressionOptions(acoid); + result->acstate = result->amroutine->cminitstate ? + result->amroutine->cminitstate(acoid, result->acoptions) : NULL; + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + remove_cached_amoptions(result); + PG_RE_THROW(); + } + PG_END_TRY(); + MemoryContextSwitchTo(oldcxt); + } + + if (!result->amroutine) + /* should not happen but need to check if something goes wrong */ + elog(ERROR, "compression access method routine is NULL"); + + return result; +} + +/* ---------- + * attr_compression_options_are_equal + * + * Compare two attribute compression records + */ +static bool +attr_compression_options_are_equal(Oid acoid1, Oid acoid2) +{ + CompressionAmOptions *a; + CompressionAmOptions *b; + + a = lookup_amoptions(acoid1); + b = lookup_amoptions(acoid2); + + if (a->amoid != b->amoid) + return false; + + if (list_length(a->acoptions) != list_length(b->acoptions)) + return false; + + return equal(a->acoptions, b->acoptions); +} diff --git a/src/backend/access/index/amapi.c b/src/backend/access/index/amapi.c index f395cb1ab4..353cd4bc39 100644 --- a/src/backend/access/index/amapi.c +++ b/src/backend/access/index/amapi.c @@ -17,6 +17,7 @@ #include "access/htup_details.h" #include "catalog/pg_am.h" #include "catalog/pg_opclass.h" +#include "commands/defrem.h" #include "utils/builtins.h" #include "utils/syscache.h" @@ -55,52 +56,12 @@ GetIndexAmRoutine(Oid amhandler) IndexAmRoutine * GetIndexAmRoutineByAmId(Oid amoid, bool noerror) { - HeapTuple tuple; - Form_pg_am amform; regproc amhandler; /* Get handler function OID for the access method */ - tuple = SearchSysCache1(AMOID, ObjectIdGetDatum(amoid)); - if (!HeapTupleIsValid(tuple)) - { - if (noerror) - return NULL; - elog(ERROR, "cache lookup failed for access method %u", - amoid); - } - amform = (Form_pg_am) GETSTRUCT(tuple); - - /* Check if it's an index access method as opposed to some other AM */ - if (amform->amtype != AMTYPE_INDEX) - { - if (noerror) - { - ReleaseSysCache(tuple); - return NULL; - } - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("access method \"%s\" is not of type %s", - NameStr(amform->amname), "INDEX"))); - } - - amhandler = amform->amhandler; - - /* Complain if handler OID is invalid */ - if (!RegProcedureIsValid(amhandler)) - { - if (noerror) - { - ReleaseSysCache(tuple); - return NULL; - } - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("index access method \"%s\" does not have a handler", - NameStr(amform->amname)))); - } - - ReleaseSysCache(tuple); + amhandler = get_am_handler_oid(amoid, AMTYPE_INDEX, noerror); + if (noerror && !RegProcedureIsValid(amhandler)) + return NULL; /* And finally, call the handler function to get the API struct. */ return GetIndexAmRoutine(amhandler); diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 28ff2f0979..87ac1a2720 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -731,6 +731,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness) attrtypes[attnum]->attcacheoff = -1; attrtypes[attnum]->atttypmod = -1; attrtypes[attnum]->attislocal = true; + attrtypes[attnum]->attcompression = InvalidOid; if (nullness == BOOTCOL_NULL_FORCE_NOT_NULL) { diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 30ca509534..eeb0438fb3 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -33,7 +33,7 @@ POSTGRES_BKI_SRCS = $(addprefix $(top_srcdir)/src/include/catalog/,\ pg_attrdef.h pg_constraint.h pg_inherits.h pg_index.h pg_operator.h \ pg_opfamily.h pg_opclass.h pg_am.h pg_amop.h pg_amproc.h \ pg_language.h pg_largeobject_metadata.h pg_largeobject.h pg_aggregate.h \ - pg_statistic_ext.h \ + pg_statistic_ext.h pg_attr_compression.h \ pg_statistic.h pg_rewrite.h pg_trigger.h pg_event_trigger.h pg_description.h \ pg_cast.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \ pg_database.h pg_db_role_setting.h pg_tablespace.h pg_pltemplate.h \ diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index b7e39af7a2..7490abcc24 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -23,6 +23,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -173,7 +174,8 @@ static const Oid object_classes[] = { PublicationRelationId, /* OCLASS_PUBLICATION */ PublicationRelRelationId, /* OCLASS_PUBLICATION_REL */ SubscriptionRelationId, /* OCLASS_SUBSCRIPTION */ - TransformRelationId /* OCLASS_TRANSFORM */ + TransformRelationId, /* OCLASS_TRANSFORM */ + AttrCompressionRelationId /* OCLASS_ATTR_COMPRESSION */ }; @@ -1283,6 +1285,10 @@ doDeletion(const ObjectAddress *object, int flags) DropTransformById(object->objectId); break; + case OCLASS_ATTR_COMPRESSION: + RemoveAttributeCompression(object->objectId); + break; + /* * These global object types are not supported here. */ @@ -2540,6 +2546,9 @@ getObjectClass(const ObjectAddress *object) case TransformRelationId: return OCLASS_TRANSFORM; + + case AttrCompressionRelationId: + return OCLASS_ATTR_COMPRESSION; } /* shouldn't get here */ diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index b69bb1e2a4..193e5da312 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -629,6 +629,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(new_attribute->attislocal); values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(new_attribute->attinhcount); values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(new_attribute->attcollation); + values[Anum_pg_attribute_attcompression - 1] = ObjectIdGetDatum(new_attribute->attcompression); /* start out with empty permissions and empty options */ nulls[Anum_pg_attribute_attacl - 1] = true; @@ -1601,6 +1602,8 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) /* We don't want to keep stats for it anymore */ attStruct->attstattarget = 0; + attStruct->attcompression = InvalidOid; + /* * Change the column name to something that isn't likely to conflict */ diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index bfac37f9d1..26c33febe5 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -411,6 +411,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->atttypmod = exprTypmod(indexkey); to->attislocal = true; to->attcollation = collationObjectId[i]; + to->attcompression = InvalidOid; ReleaseSysCache(tuple); @@ -489,6 +490,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->attbyval = typeTup->typbyval; to->attalign = typeTup->typalign; to->attstorage = typeTup->typstorage; + to->attcompression = InvalidOid; ReleaseSysCache(tuple); } diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 119297b33a..1b395e6b5a 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -23,6 +23,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -490,6 +491,18 @@ static const ObjectPropertyType ObjectProperty[] = InvalidAttrNumber, /* no ACL (same as relation) */ OBJECT_STATISTIC_EXT, true + }, + { + AttrCompressionRelationId, + AttrCompressionIndexId, + ATTCOMPRESSIONOID, + -1, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + -1, + false } }; @@ -2411,6 +2424,7 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser"))); break; + case OBJECT_STATISTIC_EXT: if (!pg_statistics_object_ownercheck(address.objectId, roleid)) aclcheck_error_type(ACLCHECK_NOT_OWNER, address.objectId); @@ -3419,6 +3433,37 @@ getObjectDescription(const ObjectAddress *object) break; } + case OCLASS_ATTR_COMPRESSION: + { + char *attname; + HeapTuple tup; + Form_pg_attr_compression acform; + + tup = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for attribute compression %u", object->objectId); + + acform = (Form_pg_attr_compression) GETSTRUCT(tup); + if (OidIsValid(acform->acrelid)) + { + appendStringInfo(&buffer, _("compression on ")); + getRelationDescription(&buffer, acform->acrelid); + + attname = get_attname(acform->acrelid, acform->acattnum, true); + if (attname) + { + appendStringInfo(&buffer, _(" column %s"), attname); + pfree(attname); + } + } + else + appendStringInfo(&buffer, _("attribute compression %u"), object->objectId); + + ReleaseSysCache(tup); + + break; + } + case OCLASS_TRANSFORM: { HeapTuple trfTup; @@ -3941,6 +3986,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "transform"); break; + case OCLASS_ATTR_COMPRESSION: + appendStringInfoString(&buffer, "attribute compression"); + break; + /* * There's intentionally no default: case here; we want the * compiler to warn if a new OCLASS hasn't been handled above. @@ -4485,6 +4534,15 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_ATTR_COMPRESSION: + { + appendStringInfo(&buffer, "%u", + object->objectId); + if (objname) + *objname = list_make1(psprintf("%u", object->objectId)); + break; + } + case OCLASS_REWRITE: { Relation ruleDesc; diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 4a6c99e090..e23abf64f1 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -13,8 +13,8 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global OBJS = amcmds.o aggregatecmds.o alter.o analyze.o async.o cluster.o comment.o \ - collationcmds.o constraint.o conversioncmds.o copy.o createas.o \ - dbcommands.o define.o discard.o dropcmds.o \ + collationcmds.o compressioncmds.o constraint.o conversioncmds.o copy.o \ + createas.o dbcommands.o define.o discard.o dropcmds.o \ event_trigger.o explain.o extension.o foreigncmds.o functioncmds.o \ indexcmds.o lockcmds.o matview.o operatorcmds.o opclasscmds.o \ policy.o portalcmds.o prepare.o proclang.o publicationcmds.o \ diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 0d63866fb0..4dda5c1690 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -631,6 +631,7 @@ AlterObjectNamespace_oid(Oid classId, Oid objid, Oid nspOid, case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: case OCLASS_TRANSFORM: + case OCLASS_ATTR_COMPRESSION: /* ignore object types that don't have schema-qualified names */ break; diff --git a/src/backend/commands/amcmds.c b/src/backend/commands/amcmds.c index f2173450ad..8f69f9e33b 100644 --- a/src/backend/commands/amcmds.c +++ b/src/backend/commands/amcmds.c @@ -175,6 +175,70 @@ get_am_type_oid(const char *amname, char amtype, bool missing_ok) return oid; } +/* + * get_am_handler_oid - return access method handler Oid with additional + * type checking. + * + * If noerror is false, throw an error if access method not found or its + * type not equal to specified type, or its handler is not valid. + * + * If amtype is not '\0', an error is raised if the AM found is not of the + * given type. + */ +regproc +get_am_handler_oid(Oid amOid, char amtype, bool noerror) +{ + HeapTuple tup; + Form_pg_am amform; + regproc amhandler; + + /* Get handler function OID for the access method */ + tup = SearchSysCache1(AMOID, ObjectIdGetDatum(amOid)); + if (!HeapTupleIsValid(tup)) + { + if (noerror) + return InvalidOid; + + elog(ERROR, "cache lookup failed for access method %u", amOid); + } + + amform = (Form_pg_am) GETSTRUCT(tup); + amhandler = amform->amhandler; + + if (amtype != '\0' && amform->amtype != amtype) + { + if (noerror) + { + ReleaseSysCache(tup); + return InvalidOid; + } + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access method \"%s\" is not of type %s", + NameStr(amform->amname), + get_am_type_string(amtype)))); + } + + /* Complain if handler OID is invalid */ + if (!RegProcedureIsValid(amhandler)) + { + if (noerror) + { + ReleaseSysCache(tup); + return InvalidOid; + } + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access method \"%s\" does not have a handler", + NameStr(amform->amname)))); + } + + ReleaseSysCache(tup); + return amhandler; +} + /* * get_index_am_oid - given an access method name, look up its OID * and verify it corresponds to an index AM. @@ -185,6 +249,16 @@ get_index_am_oid(const char *amname, bool missing_ok) return get_am_type_oid(amname, AMTYPE_INDEX, missing_ok); } +/* + * get_index_am_oid - given an access method name, look up its OID + * and verify it corresponds to an index AM. + */ +Oid +get_compression_am_oid(const char *amname, bool missing_ok) +{ + return get_am_type_oid(amname, AMTYPE_COMPRESSION, missing_ok); +} + /* * get_am_oid - given an access method name, look up its OID. * The type is not checked. @@ -225,6 +299,8 @@ get_am_type_string(char amtype) { case AMTYPE_INDEX: return "INDEX"; + case AMTYPE_COMPRESSION: + return "COMPRESSION"; default: /* shouldn't happen */ elog(ERROR, "invalid access method type '%c'", amtype); @@ -263,6 +339,14 @@ lookup_index_am_handler_func(List *handler_name, char amtype) NameListToString(handler_name), "index_am_handler"))); break; + case AMTYPE_COMPRESSION: + if (get_func_rettype(handlerOid) != COMPRESSION_AM_HANDLEROID) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must return type %s", + NameListToString(handler_name), + "compression_am_handler"))); + break; default: elog(ERROR, "unrecognized access method type \"%c\"", amtype); } diff --git a/src/backend/commands/compressioncmds.c b/src/backend/commands/compressioncmds.c new file mode 100644 index 0000000000..3a051a215e --- /dev/null +++ b/src/backend/commands/compressioncmds.c @@ -0,0 +1,659 @@ +/*------------------------------------------------------------------------- + * + * compressioncmds.c + * Routines for SQL commands for compression access methods + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/compressioncmds.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "miscadmin.h" + +#include "access/cmapi.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "access/xact.h" +#include "catalog/catalog.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_attr_compression.h" +#include "catalog/pg_am.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "parser/parse_func.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" +#include "utils/snapmgr.h" + +/* + * When conditions of compression satisfies one if builtin attribute + * compresssion tuples the compressed attribute will be linked to + * builtin compression without new record in pg_attr_compression. + * So the fact that the column has a builtin compression we only can find out + * by its dependency. + */ +static void +lookup_builtin_dependencies(Oid attrelid, AttrNumber attnum, + List **amoids) +{ + LOCKMODE lock = AccessShareLock; + Oid amoid = InvalidOid; + HeapTuple tup; + Relation rel; + SysScanDesc scan; + ScanKeyData key[3]; + + rel = heap_open(DependRelationId, lock); + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(attrelid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tup); + + if (depform->refclassid == AttrCompressionRelationId) + { + Assert(IsBuiltinCompression(depform->refobjid)); + amoid = GetAttrCompressionAmOid(depform->refobjid); + *amoids = list_append_unique_oid(*amoids, amoid); + } + } + + systable_endscan(scan); + heap_close(rel, lock); +} + +/* + * Find identical attribute compression for reuse and fill the list with + * used compression access methods. + */ +static Oid +lookup_attribute_compression(Oid attrelid, AttrNumber attnum, + Oid amoid, Datum acoptions, List **previous_amoids) +{ + Relation rel; + HeapTuple tuple; + SysScanDesc scan; + FmgrInfo arrayeq_info; + Oid result = InvalidOid; + ScanKeyData key[2]; + + /* fill FmgrInfo for array_eq function */ + fmgr_info(F_ARRAY_EQ, &arrayeq_info); + + Assert((attrelid > 0 && attnum > 0) || (attrelid == 0 && attnum == 0)); + if (previous_amoids) + lookup_builtin_dependencies(attrelid, attnum, previous_amoids); + + rel = heap_open(AttrCompressionRelationId, AccessShareLock); + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(attrelid)); + + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Oid acoid, + tup_amoid; + Datum values[Natts_pg_attr_compression]; + bool nulls[Natts_pg_attr_compression]; + + heap_deform_tuple(tuple, RelationGetDescr(rel), values, nulls); + acoid = DatumGetObjectId(values[Anum_pg_attr_compression_acoid - 1]); + tup_amoid = get_am_oid( + NameStr(*DatumGetName(values[Anum_pg_attr_compression_acname - 1])), false); + + if (previous_amoids) + *previous_amoids = list_append_unique_oid(*previous_amoids, tup_amoid); + + if (tup_amoid != amoid) + continue; + + /* + * even if we found the match, we still need to acquire all previous + * access methods so don't break cycle if previous_amoids is not NULL + */ + if (nulls[Anum_pg_attr_compression_acoptions - 1]) + { + if (DatumGetPointer(acoptions) == NULL) + result = acoid; + } + else + { + bool equal; + + /* check if arrays for WITH options are equal */ + equal = DatumGetBool(CallerFInfoFunctionCall2( + array_eq, + &arrayeq_info, + InvalidOid, + acoptions, + values[Anum_pg_attr_compression_acoptions - 1])); + if (equal) + result = acoid; + } + + if (previous_amoids == NULL && OidIsValid(result)) + break; + } + + systable_endscan(scan); + heap_close(rel, AccessShareLock); + return result; +} + +/* + * Link compression with an attribute. Creates a row in pg_attr_compression + * if needed. + * + * When compression is not specified returns default attribute compression. + * It is possible case for CREATE TABLE and ADD COLUMN commands + * where COMPRESSION syntax is optional. + * + * If any of builtin attribute compression tuples satisfies conditions + * returns it. + * + * For ALTER command check for previous attribute compression record with + * identical compression options and reuse it if found any. + * + * Note we create attribute compression for EXTERNAL storage too, so when + * storage is changed we can start compression on future tuples right away. + */ +Oid +CreateAttributeCompression(Form_pg_attribute att, + ColumnCompression *compression, + bool *need_rewrite, List **preserved_amoids) +{ + Relation rel; + HeapTuple newtup; + Oid acoid = InvalidOid, + amoid; + regproc amhandler; + Datum arropt; + Datum values[Natts_pg_attr_compression]; + bool nulls[Natts_pg_attr_compression]; + + ObjectAddress myself, + amref; + + CompressionAmRoutine *routine; + + /* No compression for PLAIN storage. */ + if (att->attstorage == 'p') + return InvalidOid; + + /* Fallback to default compression if it's not specified */ + if (compression == NULL) + return DefaultCompressionOid; + + amoid = get_compression_am_oid(compression->amname, false); + if (compression->options) + arropt = optionListToArray(compression->options, true); + else + arropt = PointerGetDatum(NULL); + + /* Try to find builtin compression first */ + acoid = lookup_attribute_compression(0, 0, amoid, arropt, NULL); + + /* + * attrelid will be invalid on CREATE TABLE, no need for table rewrite + * check. + */ + if (OidIsValid(att->attrelid)) + { + Oid attacoid; + List *previous_amoids = NIL; + + /* + * Try to find identical compression from previous tuples, and fill + * the list of previous compresssion methods + */ + attacoid = lookup_attribute_compression(att->attrelid, att->attnum, + amoid, arropt, &previous_amoids); + if (!OidIsValid(acoid)) + acoid = attacoid; + + /* + * Determine if the column needs rewrite or not. Rewrite conditions: - + * SET COMPRESSION without PRESERVE - SET COMPRESSION with PRESERVE + * but not with full list of previous access methods. + */ + if (need_rewrite != NULL) + { + /* no rewrite by default */ + *need_rewrite = false; + + Assert(preserved_amoids != NULL); + + if (compression->preserve == NIL) + { + Assert(!IsBinaryUpgrade); + *need_rewrite = true; + } + else + { + ListCell *cell; + + foreach(cell, compression->preserve) + { + char *amname_p = strVal(lfirst(cell)); + Oid amoid_p = get_am_oid(amname_p, false); + + if (!list_member_oid(previous_amoids, amoid_p)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" compression access method cannot be preserved", amname_p), + errhint("use \"pg_column_compression\" function for list of compression methods") + )); + + *preserved_amoids = list_append_unique_oid(*preserved_amoids, amoid_p); + + /* + * Remove from previous list, also protect from multiple + * mentions of one access method in PRESERVE list + */ + previous_amoids = list_delete_oid(previous_amoids, amoid_p); + } + + /* + * If the list of previous Oids is not empty after deletions + * then we need to rewrite tuples in the table. + * + * In binary upgrade list will not be free since it contains + * Oid of builtin compression access method. + */ + if (!IsBinaryUpgrade && list_length(previous_amoids) != 0) + *need_rewrite = true; + } + } + + /* Cleanup */ + list_free(previous_amoids); + } + + if (IsBinaryUpgrade && !OidIsValid(acoid)) + elog(ERROR, "could not restore attribute compression data"); + + /* Return Oid if we already found identical compression on this column */ + if (OidIsValid(acoid)) + { + if (DatumGetPointer(arropt) != NULL) + pfree(DatumGetPointer(arropt)); + + return acoid; + } + + /* Initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + amhandler = get_am_handler_oid(amoid, AMTYPE_COMPRESSION, false); + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + + acoid = GetNewOidWithIndex(rel, AttrCompressionIndexId, + Anum_pg_attr_compression_acoid); + if (acoid < FirstNormalObjectId) + { + /* this is database initialization */ + heap_close(rel, RowExclusiveLock); + return DefaultCompressionOid; + } + + /* we need routine only to call cmcheck function */ + routine = InvokeCompressionAmHandler(amhandler); + if (routine->cmcheck != NULL) + routine->cmcheck(att, compression->options); + pfree(routine); + + values[Anum_pg_attr_compression_acoid - 1] = ObjectIdGetDatum(acoid); + values[Anum_pg_attr_compression_acname - 1] = CStringGetDatum(compression->amname); + values[Anum_pg_attr_compression_acrelid - 1] = ObjectIdGetDatum(att->attrelid); + values[Anum_pg_attr_compression_acattnum - 1] = Int32GetDatum(att->attnum); + + if (compression->options) + values[Anum_pg_attr_compression_acoptions - 1] = arropt; + else + nulls[Anum_pg_attr_compression_acoptions - 1] = true; + + newtup = heap_form_tuple(RelationGetDescr(rel), values, nulls); + CatalogTupleInsert(rel, newtup); + heap_freetuple(newtup); + heap_close(rel, RowExclusiveLock); + + ObjectAddressSet(myself, AttrCompressionRelationId, acoid); + ObjectAddressSet(amref, AccessMethodRelationId, amoid); + + recordDependencyOn(&myself, &amref, DEPENDENCY_NORMAL); + recordDependencyOnCurrentExtension(&myself, false); + + /* Make the changes visible */ + CommandCounterIncrement(); + return acoid; +} + +/* + * Remove the attribute compression record from pg_attr_compression and + * call drop callback from access method routine. + */ +void +RemoveAttributeCompression(Oid acoid) +{ + Relation relation; + HeapTuple tup; + CompressionAmRoutine *routine; + Form_pg_attr_compression acform; + + tup = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tup); + + /* check we're not trying to remove builtin attribute compression */ + Assert(acform->acrelid != 0); + + /* call drop callback */ + routine = GetCompressionAmRoutine(acoid); + if (routine->cmdrop) + routine->cmdrop(acoid); + + /* delete the record from catalogs */ + relation = heap_open(AttrCompressionRelationId, RowExclusiveLock); + CatalogTupleDelete(relation, &tup->t_self); + heap_close(relation, RowExclusiveLock); + ReleaseSysCache(tup); +} + +/* + * CleanupAttributeCompression + * + * Remove entries in pg_attr_compression except current attribute compression + * and related with specified list of access methods. + */ +void +CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) +{ + Oid acoid, + amoid; + Relation rel; + SysScanDesc scan; + ScanKeyData key[3]; + HeapTuple tuple, + attrtuple; + Form_pg_attribute attform; + List *removed = NIL; + ListCell *lc; + + attrtuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + + if (!HeapTupleIsValid(attrtuple)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + attnum, relid); + attform = (Form_pg_attribute) GETSTRUCT(attrtuple); + acoid = attform->attcompression; + ReleaseSysCache(attrtuple); + + Assert(relid > 0 && attnum > 0); + + if (IsBinaryUpgrade) + goto builtin_removal; + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + + /* Remove attribute compression tuples and collect removed Oids to list */ + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_attr_compression acform; + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_am_oid(NameStr(acform->acname), false); + + /* skip current compression */ + if (acform->acoid == acoid) + continue; + + if (!list_member_oid(keepAmOids, amoid)) + { + CompressionAmRoutine *routine; + + /* call drop callback */ + routine = GetCompressionAmRoutine(acform->acoid); + if (routine->cmdrop) + routine->cmdrop(acoid); + pfree(routine); + + removed = lappend_oid(removed, acform->acoid); + CatalogTupleDelete(rel, &tuple->t_self); + } + } + + systable_endscan(scan); + heap_close(rel, RowExclusiveLock); + + /* Now remove dependencies */ + rel = heap_open(DependRelationId, RowExclusiveLock); + foreach(lc, removed) + { + Oid tup_acoid = lfirst_oid(lc); + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(AttrCompressionRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(tup_acoid)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 2, key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + CatalogTupleDelete(rel, &tuple->t_self); + + systable_endscan(scan); + } + heap_close(rel, RowExclusiveLock); + +builtin_removal: + /* Now remove dependencies with builtin compressions */ + rel = heap_open(DependRelationId, RowExclusiveLock); + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tuple); + + if (depform->refclassid != AttrCompressionRelationId) + continue; + + /* skip current compression */ + if (depform->refobjid == acoid) + continue; + + amoid = GetAttrCompressionAmOid(depform->refobjid); + if (!list_member_oid(keepAmOids, amoid)) + CatalogTupleDelete(rel, &tuple->t_self); + } + + systable_endscan(scan); + heap_close(rel, RowExclusiveLock); +} + +/* + * Construct ColumnCompression node by attribute compression Oid + */ +ColumnCompression * +MakeColumnCompression(Oid acoid) +{ + HeapTuple tuple; + Form_pg_attr_compression acform; + ColumnCompression *node; + + if (!OidIsValid(acoid)) + return NULL; + + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + node = makeNode(ColumnCompression); + node->amname = pstrdup(NameStr(acform->acname)); + ReleaseSysCache(tuple); + + /* + * fill attribute compression options too. We could've do it above but + * it's easier to call this helper. + */ + node->options = GetAttrCompressionOptions(acoid); + + return node; +} + +void +CheckCompressionMismatch(ColumnCompression *c1, ColumnCompression *c2, + const char *attributeName) +{ + if (strcmp(c1->amname, c2->amname)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression method conflict", + attributeName), + errdetail("%s versus %s", c1->amname, c2->amname))); + + if (!equal(c1->options, c2->options)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression options conflict", + attributeName), + errdetail("(%s) versus (%s)", + formatRelOptions(c1->options), + formatRelOptions(c2->options)))); +} + +Datum +pg_column_compression(PG_FUNCTION_ARGS) +{ + Oid relOid = PG_GETARG_OID(0); + char *attname = TextDatumGetCString(PG_GETARG_TEXT_P(1)); + Relation rel; + HeapTuple tuple; + AttrNumber attnum; + List *amoids = NIL; + Oid amoid; + ListCell *lc; + + ScanKeyData key[2]; + SysScanDesc scan; + StringInfoData result; + + attnum = get_attnum(relOid, attname); + if (attnum == InvalidAttrNumber) + PG_RETURN_NULL(); + + lookup_builtin_dependencies(relOid, attnum, &amoids); + + rel = heap_open(AttrCompressionRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relOid)); + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_attr_compression acform; + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_am_oid(NameStr(acform->acname), false); + amoids = list_append_unique_oid(amoids, amoid); + } + + systable_endscan(scan); + heap_close(rel, AccessShareLock); + + if (!list_length(amoids)) + PG_RETURN_NULL(); + + amoid = InvalidOid; + initStringInfo(&result); + foreach(lc, amoids) + { + if (OidIsValid(amoid)) + appendStringInfoString(&result, ", "); + + amoid = lfirst_oid(lc); + appendStringInfoString(&result, get_am_name(amoid)); + } + + PG_RETURN_TEXT_P(CStringGetTextDatum(result.data)); +} diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index a42861da0d..2148b7064b 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2516,7 +2516,7 @@ CopyFrom(CopyState cstate) values = (Datum *) palloc(tupDesc->natts * sizeof(Datum)); nulls = (bool *) palloc(tupDesc->natts * sizeof(bool)); - bistate = GetBulkInsertState(); + bistate = GetBulkInsertState(NULL); econtext = GetPerTupleExprContext(estate); /* Set up callback to identify error line number */ diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 3d82edbf58..c7059d5f62 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -569,7 +569,7 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) */ myState->hi_options = HEAP_INSERT_SKIP_FSM | (XLogIsNeeded() ? 0 : HEAP_INSERT_SKIP_WAL); - myState->bistate = GetBulkInsertState(); + myState->bistate = GetBulkInsertState(NULL); /* Not using WAL requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber); diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 549c7ea51d..f4948465ea 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1187,6 +1187,7 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: case OCLASS_TRANSFORM: + case OCLASS_ATTR_COMPRESSION: return true; /* diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 5c53aeeaeb..6db6e38a07 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -48,50 +48,6 @@ typedef struct /* Internal functions */ static void import_error_callback(void *arg); - -/* - * Convert a DefElem list to the text array format that is used in - * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and - * pg_foreign_table. - * - * Returns the array in the form of a Datum, or PointerGetDatum(NULL) - * if the list is empty. - * - * Note: The array is usually stored to database without further - * processing, hence any validation should be done before this - * conversion. - */ -static Datum -optionListToArray(List *options) -{ - ArrayBuildState *astate = NULL; - ListCell *cell; - - foreach(cell, options) - { - DefElem *def = lfirst(cell); - const char *value; - Size len; - text *t; - - value = defGetString(def); - len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); - t = palloc(len + 1); - SET_VARSIZE(t, len); - sprintf(VARDATA(t), "%s=%s", def->defname, value); - - astate = accumArrayResult(astate, PointerGetDatum(t), - false, TEXTOID, - CurrentMemoryContext); - } - - if (astate) - return makeArrayResult(astate, CurrentMemoryContext); - - return PointerGetDatum(NULL); -} - - /* * Transform a list of DefElem into text array format. This is substantially * the same thing as optionListToArray(), except we recognize SET/ADD/DROP @@ -178,7 +134,7 @@ transformGenericOptions(Oid catalogId, } } - result = optionListToArray(resultOptions); + result = optionListToArray(resultOptions, false); if (OidIsValid(fdwvalidator)) { diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 23892b1b81..2802e33402 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -465,7 +465,7 @@ transientrel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) myState->hi_options = HEAP_INSERT_SKIP_FSM | HEAP_INSERT_FROZEN; if (!XLogIsNeeded()) myState->hi_options |= HEAP_INSERT_SKIP_WAL; - myState->bistate = GetBulkInsertState(); + myState->bistate = GetBulkInsertState(NULL); /* Not using WAL requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(transientrel) == InvalidBlockNumber); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index e74fb1f469..69ad7be718 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -21,6 +21,7 @@ #include "access/relscan.h" #include "access/sysattr.h" #include "access/tupconvert.h" +#include "access/tuptoaster.h" #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" @@ -32,6 +33,7 @@ #include "catalog/objectaccess.h" #include "catalog/partition.h" #include "catalog/pg_am.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_constraint_fn.h" @@ -41,6 +43,7 @@ #include "catalog/pg_inherits_fn.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" +#include "catalog/pg_proc.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" @@ -90,6 +93,7 @@ #include "storage/smgr.h" #include "utils/acl.h" #include "utils/builtins.h" +#include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" @@ -162,6 +166,8 @@ typedef struct AlteredTableInfo /* Information saved by Phases 1/2 for Phase 3: */ List *constraints; /* List of NewConstraint */ List *newvals; /* List of NewColumnValue */ + HTAB *preservedAmInfo; /* Hash table for preserved compression + * methods */ bool new_notnull; /* T if we added new NOT NULL constraints */ int rewrite; /* Reason for forced rewrite, if any */ Oid newTableSpace; /* new tablespace; 0 means no change */ @@ -370,6 +376,8 @@ static bool check_for_column_name_collision(Relation rel, const char *colname, bool if_not_exists); static void add_column_datatype_dependency(Oid relid, int32 attnum, Oid typid); static void add_column_collation_dependency(Oid relid, int32 attnum, Oid collid); +static void add_column_compression_dependency(Oid relid, int32 attnum, Oid acoid); +static void set_column_compression_relid(Oid acoid, Oid relid); static void ATPrepAddOids(List **wqueue, Relation rel, bool recurse, AlterTableCmd *cmd, LOCKMODE lockmode); static void ATPrepDropNotNull(Relation rel, bool recurse, bool recursing); @@ -466,6 +474,8 @@ static void ATExecGenericOptions(Relation rel, List *options); static void ATExecEnableRowSecurity(Relation rel); static void ATExecDisableRowSecurity(Relation rel); static void ATExecForceNoForceRowSecurity(Relation rel, bool force_rls); +static ObjectAddress ATExecSetCompression(AlteredTableInfo *tab, Relation rel, + const char *column, ColumnCompression *compression, LOCKMODE lockmode); static void copy_relation_data(SMgrRelation rel, SMgrRelation dst, ForkNumber forkNum, char relpersistence); @@ -518,6 +528,7 @@ ObjectAddress DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, ObjectAddress *typaddress, const char *queryString) { + int i; char relname[NAMEDATALEN]; Oid namespaceId; Oid relationId; @@ -692,6 +703,8 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, * We can set the atthasdef flags now in the tuple descriptor; this just * saves StoreAttrDefault from having to do an immediate update of the * pg_attribute rows. + * + * Also create attribute compression records if needed. */ rawDefaults = NIL; cookedDefaults = NIL; @@ -737,6 +750,13 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (colDef->identity) attr->attidentity = colDef->identity; + + if (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE) + attr->attcompression = CreateAttributeCompression(attr, + colDef->compression, + NULL, NULL); + else + attr->attcompression = InvalidOid; } /* @@ -785,6 +805,23 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, */ rel = relation_open(relationId, AccessExclusiveLock); + /* + * Specify relation Oid in attribute compression tuples, and create + * dependencies. + */ + for (i = 0; i < (RelationGetDescr(rel))->natts; i++) + { + Form_pg_attribute attr; + + attr = TupleDescAttr(RelationGetDescr(rel), i); + if (OidIsValid(attr->attcompression)) + { + set_column_compression_relid(attr->attcompression, relationId); + add_column_compression_dependency(attr->attrelid, attr->attnum, + attr->attcompression); + } + } + /* Process and store partition bound, if any. */ if (stmt->partbound) { @@ -1677,6 +1714,7 @@ storage_name(char c) } } + /*---------- * MergeAttributes * Returns new schema given initial schema and superclasses. @@ -2011,6 +2049,19 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(attribute->attstorage)))); + if (OidIsValid(attribute->attcompression)) + { + ColumnCompression *compression = + MakeColumnCompression(attribute->attcompression); + + if (!def->compression) + def->compression = compression; + else + CheckCompressionMismatch(def->compression, + compression, + attributeName); + } + def->inhcount++; /* Merge of NOT NULL constraints = OR 'em together */ def->is_not_null |= attribute->attnotnull; @@ -2038,6 +2089,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collOid = attribute->attcollation; def->constraints = NIL; def->location = -1; + def->compression = MakeColumnCompression(attribute->attcompression); inhSchema = lappend(inhSchema, def); newattno[parent_attno - 1] = ++child_attno; } @@ -2247,6 +2299,13 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(newdef->storage)))); + if (!def->compression) + def->compression = newdef->compression; + else if (newdef->compression) + CheckCompressionMismatch(def->compression, + newdef->compression, + attributeName); + /* Mark the column as locally defined */ def->is_local = true; /* Merge of NOT NULL constraints = OR 'em together */ @@ -3333,6 +3392,7 @@ AlterTableGetLockLevel(List *cmds) */ case AT_GenericOptions: case AT_AlterColumnGenericOptions: + case AT_SetCompression: cmd_lockmode = AccessExclusiveLock; break; @@ -3808,6 +3868,7 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: + case AT_SetCompression: ATSimplePermissions(rel, ATT_TABLE); /* These commands never recurse */ /* No command-specific prep needed */ @@ -4182,6 +4243,11 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); ATExecDetachPartition(rel, ((PartitionCmd *) cmd->def)->name); break; + case AT_SetCompression: + address = ATExecSetCompression(tab, rel, cmd->name, + (ColumnCompression *) cmd->def, + lockmode); + break; default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -4247,8 +4313,9 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode) /* * We only need to rewrite the table if at least one column needs to - * be recomputed, we are adding/removing the OID column, or we are - * changing its persistence. + * be recomputed, we are adding/removing the OID column, we are + * changing its persistence, or we are changing compression type of a + * column without preserving old ones. * * There are two reasons for requiring a rewrite when changing * persistence: on one hand, we need to ensure that the buffers @@ -4477,7 +4544,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode) if (newrel) { mycid = GetCurrentCommandId(true); - bistate = GetBulkInsertState(); + bistate = GetBulkInsertState(tab->preservedAmInfo); hi_options = HEAP_INSERT_SKIP_FSM; if (!XLogIsNeeded()) @@ -4748,6 +4815,24 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode) FreeExecutorState(estate); + /* Remove old compression options */ + if (tab->rewrite & AT_REWRITE_ALTER_COMPRESSION) + { + AttrCmPreservedInfo *pinfo; + HASH_SEQ_STATUS status; + + Assert(tab->preservedAmInfo); + hash_seq_init(&status, tab->preservedAmInfo); + while ((pinfo = (AttrCmPreservedInfo *) hash_seq_search(&status)) != NULL) + { + CleanupAttributeCompression(tab->relid, pinfo->attnum, + pinfo->preserved_amoids); + list_free(pinfo->preserved_amoids); + } + + hash_destroy(tab->preservedAmInfo); + } + heap_close(oldrel, NoLock); if (newrel) { @@ -5409,6 +5494,16 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.attislocal = colDef->is_local; attribute.attinhcount = colDef->inhcount; attribute.attcollation = collOid; + + /* create attribute compresssion record */ + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + attribute.attcompression = CreateAttributeCompression(&attribute, + colDef->compression, + NULL, NULL); + else + attribute.attcompression = InvalidOid; + /* attribute.attacl is handled by InsertPgAttributeTuple */ ReleaseSysCache(typeTuple); @@ -5561,6 +5656,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, */ add_column_datatype_dependency(myrelid, newattnum, attribute.atttypid); add_column_collation_dependency(myrelid, newattnum, attribute.attcollation); + add_column_compression_dependency(myrelid, newattnum, attribute.attcompression); /* * Propagate to children as appropriate. Unlike most other ALTER @@ -5683,6 +5779,30 @@ add_column_datatype_dependency(Oid relid, int32 attnum, Oid typid) recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); } +/* Update acrelid in pg_attr_compression */ +static void +set_column_compression_relid(Oid acoid, Oid relid) +{ + Relation rel; + HeapTuple tuple; + Form_pg_attr_compression acform; + + Assert(OidIsValid(relid)); + if (IsBuiltinCompression(acoid)) + return; + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + acform->acrelid = relid; + CatalogTupleUpdate(rel, &tuple->t_self, tuple); + ReleaseSysCache(tuple); + heap_close(rel, RowExclusiveLock); +} + /* * Install a column's dependency on its collation. */ @@ -5705,6 +5825,106 @@ add_column_collation_dependency(Oid relid, int32 attnum, Oid collid) } } +/* + * Install a dependency for compression options on its column. + * + * If it is builtin attribute compression then column depends on it. This + * is used to determine connection between column and builtin attribute + * compression in ALTER SET COMPRESSION command. + * + * If dependency is already there the whole thing is skipped. + */ +static void +add_column_compression_dependency(Oid relid, int32 attnum, Oid acoid) +{ + bool found = false; + Relation depRel; + ObjectAddress acref, + attref; + HeapTuple depTup; + ScanKeyData key[3]; + SysScanDesc scan; + + if (!OidIsValid(acoid)) + return; + + Assert(relid > 0 && attnum > 0); + ObjectAddressSet(acref, AttrCompressionRelationId, acoid); + ObjectAddressSubSet(attref, RelationRelationId, relid, attnum); + + depRel = heap_open(DependRelationId, AccessShareLock); + + if (IsBuiltinCompression(acoid)) + { + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(depRel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(depTup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(depTup); + + if (foundDep->refclassid == AttrCompressionRelationId && + foundDep->refobjid == acoid) + { + found = true; + break; + } + } + systable_endscan(scan); + + if (!found) + recordDependencyOn(&attref, &acref, DEPENDENCY_NORMAL); + } + else + { + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_refobjsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(depTup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(depTup); + + if (foundDep->classid == AttrCompressionRelationId && + foundDep->objid == acoid) + { + found = true; + break; + } + } + systable_endscan(scan); + + if (!found) + recordDependencyOn(&acref, &attref, DEPENDENCY_INTERNAL); + } + heap_close(depRel, AccessShareLock); +} + /* * ALTER TABLE SET WITH OIDS * @@ -6540,6 +6760,10 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc errmsg("column data type %s can only have storage PLAIN", format_type_be(attrtuple->atttypid)))); + /* use default compression if storage is not PLAIN */ + if (!OidIsValid(attrtuple->attcompression) && (newstorage != 'p')) + attrtuple->attcompression = DefaultCompressionOid; + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); InvokeObjectPostAlterHook(RelationRelationId, @@ -8601,6 +8825,45 @@ createForeignKeyTriggers(Relation rel, Oid refRelOid, Constraint *fkconstraint, indexOid, false); } +/* + * Initialize hash table used to keep rewrite rules for + * compression changes in ALTER commands. + */ +static void +setupCompressionRewriteRules(AlteredTableInfo *tab, const char *column, + AttrNumber attnum, List *preserved_amoids) +{ + bool found; + AttrCmPreservedInfo *pinfo; + + Assert(!IsBinaryUpgrade); + tab->rewrite |= AT_REWRITE_ALTER_COMPRESSION; + + /* initialize hash for oids */ + if (tab->preservedAmInfo == NULL) + { + HASHCTL ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(AttrNumber); + ctl.entrysize = sizeof(AttrCmPreservedInfo); + tab->preservedAmInfo = + hash_create("preserved access methods cache", 10, &ctl, + HASH_ELEM | HASH_BLOBS); + } + pinfo = (AttrCmPreservedInfo *) hash_search(tab->preservedAmInfo, + &attnum, HASH_ENTER, &found); + + if (found) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter compression of column \"%s\" twice", column), + errhint("Remove one of statements from the command."))); + + pinfo->attnum = attnum; + pinfo->preserved_amoids = preserved_amoids; +} + /* * ALTER TABLE DROP CONSTRAINT * @@ -9356,6 +9619,12 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, colName))); break; + case OCLASS_ATTR_COMPRESSION: + + /* Just check that dependency is the right type */ + Assert(foundDep->deptype == DEPENDENCY_INTERNAL); + break; + case OCLASS_DEFAULT: /* @@ -9456,7 +9725,8 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, if (!(foundDep->refclassid == TypeRelationId && foundDep->refobjid == attTup->atttypid) && !(foundDep->refclassid == CollationRelationId && - foundDep->refobjid == attTup->attcollation)) + foundDep->refobjid == attTup->attcollation) && + foundDep->refclassid != AttrCompressionRelationId) elog(ERROR, "found unexpected dependency for column"); CatalogTupleDelete(depRel, &depTup->t_self); @@ -9481,6 +9751,34 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, ReleaseSysCache(typeTuple); + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* Setup attribute compression for new attribute */ + if (OidIsValid(attTup->attcompression) && attTup->attstorage == 'p') + { + /* Set up rewrite of table */ + attTup->attcompression = InvalidOid; + setupCompressionRewriteRules(tab, colName, attOldTup->attnum, NIL); + } + else if (OidIsValid(attTup->attcompression)) + { + /* create new record */ + ColumnCompression *compression = MakeColumnCompression(attTup->attcompression); + + if (!IsBuiltinCompression(attTup->attcompression)) + setupCompressionRewriteRules(tab, colName, attOldTup->attnum, NIL); + + attTup->attcompression = CreateAttributeCompression(attTup, compression, NULL, NULL); + } + else if (attTup->attstorage != 'p') + attTup->attcompression = DefaultCompressionOid; + else + attTup->attcompression = InvalidOid; + } + else + attTup->attcompression = InvalidOid; + CatalogTupleUpdate(attrelation, &heapTup->t_self, heapTup); heap_close(attrelation, RowExclusiveLock); @@ -9489,6 +9787,11 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, add_column_datatype_dependency(RelationGetRelid(rel), attnum, targettype); add_column_collation_dependency(RelationGetRelid(rel), attnum, targetcollid); + /* Create dependency for new attribute compression */ + if (OidIsValid(attTup->attcompression)) + add_column_compression_dependency(RelationGetRelid(rel), attnum, + attTup->attcompression); + /* * Drop any pg_statistic entry for the column, since it's now wrong type */ @@ -12517,6 +12820,94 @@ ATExecGenericOptions(Relation rel, List *options) heap_freetuple(tuple); } +static ObjectAddress +ATExecSetCompression(AlteredTableInfo *tab, + Relation rel, + const char *column, + ColumnCompression *compression, + LOCKMODE lockmode) +{ + Oid acoid; + Relation attrel; + HeapTuple atttuple; + Form_pg_attribute atttableform; + AttrNumber attnum; + bool need_rewrite; + Datum values[Natts_pg_attribute]; + bool nulls[Natts_pg_attribute]; + bool replace[Natts_pg_attribute]; + List *preserved_amoids = NIL; + ObjectAddress address; + + attrel = heap_open(AttributeRelationId, RowExclusiveLock); + + atttuple = SearchSysCacheAttName(RelationGetRelid(rel), column); + if (!HeapTupleIsValid(atttuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + column, RelationGetRelationName(rel)))); + + /* Prevent them from altering a system attribute */ + atttableform = (Form_pg_attribute) GETSTRUCT(atttuple); + attnum = atttableform->attnum; + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", column))); + + /* Prevent from altering untoastable attributes with PLAIN storage */ + if (atttableform->attstorage == 'p' && !TypeIsToastable(atttableform->atttypid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column data type %s does not support compression", + format_type_be(atttableform->atttypid)))); + + /* Initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replace, false, sizeof(replace)); + + /* create pg_attr_compression record and its dependencies */ + acoid = CreateAttributeCompression(atttableform, compression, + &need_rewrite, &preserved_amoids); + add_column_compression_dependency(atttableform->attrelid, + atttableform->attnum, acoid); + + /* + * Save list of preserved access methods to cache which will be passed to + * toast_insert_or_update + */ + if (need_rewrite) + setupCompressionRewriteRules(tab, column, atttableform->attnum, + preserved_amoids); + + atttableform->attcompression = acoid; + CatalogTupleUpdate(attrel, &atttuple->t_self, atttuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + atttableform->attnum); + + ReleaseSysCache(atttuple); + heap_close(attrel, RowExclusiveLock); + + /* make changes visible */ + CommandCounterIncrement(); + + /* + * Normally cleanup is done in rewrite but in binary upgrade we should do + * it explicitly. + */ + if (IsBinaryUpgrade) + CleanupAttributeCompression(RelationGetRelid(rel), + attnum, preserved_amoids); + + ObjectAddressSet(address, AttrCompressionRelationId, acoid); + return address; +} + + /* * Preparation phase for SET LOGGED/UNLOGGED * diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index c7293a60d7..e331f19448 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2821,6 +2821,7 @@ _copyColumnDef(const ColumnDef *from) COPY_STRING_FIELD(colname); COPY_NODE_FIELD(typeName); + COPY_NODE_FIELD(compression); COPY_SCALAR_FIELD(inhcount); COPY_SCALAR_FIELD(is_local); COPY_SCALAR_FIELD(is_not_null); @@ -2840,6 +2841,18 @@ _copyColumnDef(const ColumnDef *from) return newnode; } +static ColumnCompression * +_copyColumnCompression(const ColumnCompression *from) +{ + ColumnCompression *newnode = makeNode(ColumnCompression); + + COPY_STRING_FIELD(amname); + COPY_NODE_FIELD(options); + COPY_NODE_FIELD(preserve); + + return newnode; +} + static Constraint * _copyConstraint(const Constraint *from) { @@ -5507,6 +5520,9 @@ copyObjectImpl(const void *from) case T_ColumnDef: retval = _copyColumnDef(from); break; + case T_ColumnCompression: + retval = _copyColumnCompression(from); + break; case T_Constraint: retval = _copyConstraint(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 765b1be74b..25f2ce9ded 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2558,6 +2558,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) { COMPARE_STRING_FIELD(colname); COMPARE_NODE_FIELD(typeName); + COMPARE_NODE_FIELD(compression); COMPARE_SCALAR_FIELD(inhcount); COMPARE_SCALAR_FIELD(is_local); COMPARE_SCALAR_FIELD(is_not_null); @@ -2577,6 +2578,16 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) return true; } +static bool +_equalColumnCompression(const ColumnCompression *a, const ColumnCompression *b) +{ + COMPARE_STRING_FIELD(amname); + COMPARE_NODE_FIELD(options); + COMPARE_NODE_FIELD(preserve); + + return true; +} + static bool _equalConstraint(const Constraint *a, const Constraint *b) { @@ -3638,6 +3649,9 @@ equal(const void *a, const void *b) case T_ColumnDef: retval = _equalColumnDef(a, b); break; + case T_ColumnCompression: + retval = _equalColumnCompression(a, b); + break; case T_Constraint: retval = _equalConstraint(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 6c76c41ebe..92b53e873e 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -3623,6 +3623,8 @@ raw_expression_tree_walker(Node *node, if (walker(coldef->typeName, context)) return true; + if (walker(coldef->compression, context)) + return true; if (walker(coldef->raw_default, context)) return true; if (walker(coldef->collClause, context)) @@ -3630,6 +3632,14 @@ raw_expression_tree_walker(Node *node, /* for now, constraints are ignored */ } break; + case T_ColumnCompression: + { + ColumnCompression *colcmp = (ColumnCompression *) node; + + if (walker(colcmp->options, context)) + return true; + } + break; case T_IndexElem: { IndexElem *indelem = (IndexElem *) node; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index f61ae03ac5..de061ecbea 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2812,6 +2812,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_STRING_FIELD(colname); WRITE_NODE_FIELD(typeName); + WRITE_NODE_FIELD(compression); WRITE_INT_FIELD(inhcount); WRITE_BOOL_FIELD(is_local); WRITE_BOOL_FIELD(is_not_null); @@ -2829,6 +2830,16 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_LOCATION_FIELD(location); } +static void +_outColumnCompression(StringInfo str, const ColumnCompression *node) +{ + WRITE_NODE_TYPE("COLUMNCOMPRESSION"); + + WRITE_STRING_FIELD(amname); + WRITE_NODE_FIELD(options); + WRITE_NODE_FIELD(preserve); +} + static void _outTypeName(StringInfo str, const TypeName *node) { @@ -4125,6 +4136,9 @@ outNode(StringInfo str, const void *obj) case T_ColumnDef: _outColumnDef(str, obj); break; + case T_ColumnCompression: + _outColumnCompression(str, obj); + break; case T_TypeName: _outTypeName(str, obj); break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index cd5ba2d4d8..99f3d1a6a7 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -311,6 +311,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); opt_grant_grant_option opt_grant_admin_option opt_nowait opt_if_exists opt_with_data %type opt_nowait_or_skip +%type AccessMethodType %type OptRoleList AlterOptRoleList %type CreateOptRoleElem AlterOptRoleElem @@ -398,6 +399,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); transform_element_list transform_type_list TriggerTransitions TriggerReferencing publication_name_list + optCompressionParameters vacuum_relation_list opt_vacuum_relation_list %type group_by_list @@ -584,6 +586,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type hash_partbound partbound_datum_list range_datum_list %type hash_partbound_elem +%type optColumnCompression alterColumnCompression +%type compressionClause +%type optCompressionPreserve + /* * Non-keyword token types. These are hard-wired into the "flex" lexer. * They must be listed first so that their numeric codes do not depend on @@ -616,9 +622,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT - COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT - CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE - CROSS CSV CUBE CURRENT_P + COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT + CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY + COST CREATE CROSS CSV CUBE CURRENT_P CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE @@ -2213,6 +2219,15 @@ alter_table_cmd: n->missing_ok = true; $$ = (Node *)n; } + /* ALTER TABLE ALTER [COLUMN] SET (COMPRESSION [WITH ()]) */ + | ALTER opt_column ColId SET alterColumnCompression + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_SetCompression; + n->name = $3; + n->def = $5; + $$ = (Node *)n; + } /* ALTER TABLE DROP [COLUMN] IF EXISTS [RESTRICT|CASCADE] */ | DROP opt_column IF_P EXISTS ColId opt_drop_behavior { @@ -3377,11 +3392,12 @@ TypedTableElement: | TableConstraint { $$ = $1; } ; -columnDef: ColId Typename create_generic_options ColQualList +columnDef: ColId Typename optColumnCompression create_generic_options ColQualList { ColumnDef *n = makeNode(ColumnDef); n->colname = $1; n->typeName = $2; + n->compression = (ColumnCompression *) $3; n->inhcount = 0; n->is_local = true; n->is_not_null = false; @@ -3391,8 +3407,8 @@ columnDef: ColId Typename create_generic_options ColQualList n->raw_default = NULL; n->cooked_default = NULL; n->collOid = InvalidOid; - n->fdwoptions = $3; - SplitColQualList($4, &n->constraints, &n->collClause, + n->fdwoptions = $4; + SplitColQualList($5, &n->constraints, &n->collClause, yyscanner); n->location = @1; $$ = (Node *)n; @@ -3439,6 +3455,43 @@ columnOptions: ColId ColQualList } ; +optCompressionPreserve: + PRESERVE '(' name_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + +optCompressionParameters: + WITH '(' generic_option_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + +compressionClause: + COMPRESSION name { $$ = pstrdup($2); } + ; + +optColumnCompression: + compressionClause optCompressionParameters + { + ColumnCompression *n = makeNode(ColumnCompression); + n->amname = $1; + n->options = (List *) $2; + n->preserve = NIL; + $$ = (Node *) n; + } + | /*EMPTY*/ { $$ = NULL; } + ; + +alterColumnCompression: + compressionClause optCompressionParameters optCompressionPreserve + { + ColumnCompression *n = makeNode(ColumnCompression); + n->amname = $1; + n->options = (List *) $2; + n->preserve = (List *) $3; + $$ = (Node *) n; + } + ; + ColQualList: ColQualList ColConstraint { $$ = lappend($1, $2); } | /*EMPTY*/ { $$ = NIL; } @@ -3644,6 +3697,7 @@ TableLikeOption: | INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; } | STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; } | STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; } + | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; } | ALL { $$ = CREATE_TABLE_LIKE_ALL; } ; @@ -5312,12 +5366,17 @@ row_security_cmd: * *****************************************************************************/ -CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name +AccessMethodType: + INDEX { $$ = AMTYPE_INDEX; } + | COMPRESSION { $$ = AMTYPE_COMPRESSION; } + ; + +CreateAmStmt: CREATE ACCESS METHOD name TYPE_P AccessMethodType HANDLER handler_name { CreateAmStmt *n = makeNode(CreateAmStmt); n->amname = $4; n->handler_name = $8; - n->amtype = AMTYPE_INDEX; + n->amtype = $6; $$ = (Node *) n; } ; @@ -14989,6 +15048,7 @@ unreserved_keyword: | COMMENTS | COMMIT | COMMITTED + | COMPRESSION | CONFIGURATION | CONFLICT | CONNECTION diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 0fd14f43c6..cfcfde5683 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -1073,6 +1073,13 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla else def->storage = 0; + /* Likewise, copy compression if requested */ + if (table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION + && OidIsValid(attribute->attcompression)) + def->compression = MakeColumnCompression(attribute->attcompression); + else + def->compression = NULL; + /* Likewise, copy comment if requested */ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) && (comment = GetComment(attribute->attrelid, diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 5ffe638b19..dce56f7bcd 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2883,7 +2883,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == toast_pointer.va_extsize); + Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)); /* make sure its marked as compressed or not */ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) diff --git a/src/backend/utils/adt/expandedrecord.c b/src/backend/utils/adt/expandedrecord.c index 0bf5fe8cc7..3668310c2a 100644 --- a/src/backend/utils/adt/expandedrecord.c +++ b/src/backend/utils/adt/expandedrecord.c @@ -795,6 +795,7 @@ ER_flatten_into(ExpandedObjectHeader *eohptr, (char *) tuphdr + erh->hoff, erh->data_len, &tuphdr->t_infomask, + &tuphdr->t_infomask2, (erh->hasnull ? tuphdr->t_bits : NULL)); } diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index dbe67cdb4c..4a97099cd0 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -418,3 +418,4 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(internal); PSEUDOTYPE_DUMMY_IO_FUNCS(opaque); PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement); PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray); +PSEUDOTYPE_DUMMY_IO_FUNCS(compression_am_handler); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 6ab4db26bd..42178ecfb4 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -565,6 +565,11 @@ RelationBuildTupleDesc(Relation relation) attrdef[ndef].adbin = NULL; ndef++; } + + /* mark tupledesc as it contains attributes with custom compression */ + if (attp->attcompression) + relation->rd_att->tdflags |= TD_ATTR_CUSTOM_COMPRESSED; + need--; if (need == 0) break; diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index 2b381782a3..0c97851cbd 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -27,6 +27,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -187,6 +188,17 @@ static const struct cachedesc cacheinfo[] = { }, 16 }, + {AttrCompressionRelationId, /* ATTCOMPRESSIONOID */ + AttrCompressionIndexId, + 1, + { + Anum_pg_attr_compression_acoid, + 0, + 0, + 0 + }, + 8, + }, {AttributeRelationId, /* ATTNAME */ AttributeRelidNameIndexId, 2, diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index ceedd481fb..b33fcbc168 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -78,6 +78,7 @@ typedef struct _restoreOptions int no_publications; /* Skip publication entries */ int no_security_labels; /* Skip security label entries */ int no_subscriptions; /* Skip subscription entries */ + int no_compression_methods; /* Skip compression methods */ int strict_names; const char *filename; @@ -122,6 +123,7 @@ typedef struct _restoreOptions bool *idWanted; /* array showing which dump IDs to emit */ int enable_row_security; int sequence_data; /* dump sequence data even in schema-only mode */ + int compression_data; /* dump attribute compression table */ int binary_upgrade; } RestoreOptions; @@ -151,6 +153,7 @@ typedef struct _dumpOptions int no_security_labels; int no_publications; int no_subscriptions; + int no_compression_methods; int no_synchronized_snapshots; int no_unlogged_table_data; int serializable_deferrable; @@ -172,6 +175,8 @@ typedef struct _dumpOptions char *outputSuperuser; int sequence_data; /* dump sequence data even in schema-only mode */ + int compression_data; /* dump compression options even in + * schema-only mode */ } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 83c976eaf7..319d78b548 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -180,6 +180,7 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->include_everything = ropt->include_everything; dopt->enable_row_security = ropt->enable_row_security; dopt->sequence_data = ropt->sequence_data; + dopt->compression_data = ropt->compression_data; return dopt; } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index b8d65a9ee3..23db7eac21 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -40,11 +40,13 @@ #include "getopt_long.h" #include "access/attnum.h" +#include "access/cmapi.h" #include "access/sysattr.h" #include "access/transam.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_attribute.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_cast.h" #include "catalog/pg_class.h" #include "catalog/pg_default_acl.h" @@ -377,6 +379,7 @@ main(int argc, char **argv) {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, + {"no-compression-methods", no_argument, &dopt.no_compression_methods, 1}, {"no-sync", no_argument, NULL, 7}, {NULL, 0, NULL, 0} @@ -597,6 +600,9 @@ main(int argc, char **argv) if (dopt.binary_upgrade) dopt.sequence_data = 1; + if (dopt.binary_upgrade) + dopt.compression_data = 1; + if (dopt.dataOnly && dopt.schemaOnly) { write_msg(NULL, "options -s/--schema-only and -a/--data-only cannot be used together\n"); @@ -808,6 +814,9 @@ main(int argc, char **argv) if (dopt.schemaOnly && dopt.sequence_data) getTableData(&dopt, tblinfo, numTables, dopt.oids, RELKIND_SEQUENCE); + if (dopt.schemaOnly && dopt.compression_data) + getAttributeCompressionData(fout); + /* * In binary-upgrade mode, we do not have to worry about the actual blob * data or the associated metadata that resides in the pg_largeobject and @@ -4195,6 +4204,233 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) free(qsubname); } +/* + * getAttributeCompressionData + * get information from pg_attr_compression + * this information should be migrated at binary upgrade + */ +void +getAttributeCompressionData(Archive *fout) +{ + PQExpBuffer query; + PGresult *res; + AttributeCompressionInfo *cminfo; + int i_tableoid; + int i_oid; + int i_relid; + int i_attnum; + int i_name; + int i_options; + int i_iscurrent; + int i_attname; + int i_parsedoptions; + int i_preserve; + int i_cnt; + int i, + j, + k, + attcnt, + ntups; + + if (fout->remoteVersion < 110000) + return; + + res = ExecuteSqlQuery(fout, + "SELECT tableoid, acrelid::pg_catalog.regclass as acrelid, acattnum, COUNT(*) AS cnt " + "FROM pg_catalog.pg_attr_compression " + "WHERE acrelid > 0 AND acattnum > 0 " + "GROUP BY tableoid, acrelid, acattnum;", + PGRES_TUPLES_OK); + + attcnt = PQntuples(res); + i_tableoid = PQfnumber(res, "tableoid"); + i_relid = PQfnumber(res, "acrelid"); + i_attnum = PQfnumber(res, "acattnum"); + i_cnt = PQfnumber(res, "cnt"); + + cminfo = pg_malloc(attcnt * sizeof(AttributeCompressionInfo)); + for (i = 0; i < attcnt; i++) + { + char *acrelid = pg_strdup(PQgetvalue(res, i, i_relid)); + uint16 attnum = atoi(PQgetvalue(res, i, i_attnum)); + int cnt = atoi(PQgetvalue(res, i, i_cnt)); + + cminfo[i].dobj.catId.tableoid = atooid(PQgetvalue(res, i, i_tableoid)); + cminfo[i].dobj.dump = DUMP_COMPONENT_DEFINITION; + cminfo[i].dobj.objType = DO_ATTRIBUTE_COMPRESSION; + AssignDumpId(&cminfo[i].dobj); + + cminfo[i].acrelid = acrelid; + cminfo[i].acattnum = attnum; + cminfo[i].preserve = NULL; + cminfo[i].attname = NULL; + cminfo[i].nitems = cnt; + cminfo[i].items = pg_malloc0(sizeof(AttributeCompressionItem) * cnt); + cminfo[i].dobj.name = psprintf("%s.%d", acrelid, attnum); + } + PQclear(res); + + query = createPQExpBuffer(); + resetPQExpBuffer(query); + appendPQExpBuffer(query, + "SELECT c.acoid, c.acname, c.acattnum," + " c.acrelid::pg_catalog.regclass as acrelid, c.acoptions," + " a.attcompression = c.acoid as iscurrent, a.attname," + " pg_catalog.pg_column_compression(a.attrelid, a.attname) as preserve," + " pg_catalog.array_to_string(ARRAY(" + " SELECT pg_catalog.quote_ident(option_name) || " + " ' ' || pg_catalog.quote_literal(option_value) " + " FROM pg_catalog.pg_options_to_table(c.acoptions) " + " ORDER BY option_name" + " ), E',\n ') AS parsedoptions " + "FROM pg_catalog.pg_attr_compression c " + "JOIN pg_attribute a ON (c.acrelid = a.attrelid " + " AND c.acattnum = a.attnum) " + "WHERE acrelid > 0 AND attnum > 0 " + "ORDER BY iscurrent"); + + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + + ntups = PQntuples(res); + + i_oid = PQfnumber(res, "acoid"); + i_name = PQfnumber(res, "acname"); + i_relid = PQfnumber(res, "acrelid"); + i_attnum = PQfnumber(res, "acattnum"); + i_options = PQfnumber(res, "acoptions"); + i_iscurrent = PQfnumber(res, "iscurrent"); + i_attname = PQfnumber(res, "attname"); + i_parsedoptions = PQfnumber(res, "parsedoptions"); + i_preserve = PQfnumber(res, "preserve"); + + for (i = 0; i < ntups; i++) + { + char *acrelid = pg_strdup(PQgetvalue(res, i, i_relid)); + uint16 attnum = atoi(PQgetvalue(res, i, i_attnum)); + + for (j = 0; j < attcnt; j++) + { + if (strcmp(cminfo[j].acrelid, acrelid) == 0 && + cminfo[j].acattnum == attnum) + { + /* in the end it will be current */ + cminfo[j].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid)); + + if (cminfo[j].preserve == NULL) + cminfo[j].preserve = pg_strdup(PQgetvalue(res, i, i_preserve)); + if (cminfo[j].attname == NULL) + cminfo[j].attname = pg_strdup(PQgetvalue(res, i, i_attname)); + + for (k = 0; k < cminfo[j].nitems; k++) + { + AttributeCompressionItem *item = &cminfo[j].items[k]; + + if (item->acname == NULL) + { + item->acoid = atooid(PQgetvalue(res, i, i_oid)); + item->acname = pg_strdup(PQgetvalue(res, i, i_name)); + item->acoptions = pg_strdup(PQgetvalue(res, i, i_options)); + item->iscurrent = PQgetvalue(res, i, i_iscurrent)[0] == 't'; + item->parsedoptions = pg_strdup(PQgetvalue(res, i, i_parsedoptions)); + + /* to next tuple */ + break; + } + } + + /* to next tuple */ + break; + } + } + } + + PQclear(res); + destroyPQExpBuffer(query); +} + +/* + * dumpAttributeCompression + * dump the given compression options + */ +static void +dumpAttributeCompression(Archive *fout, AttributeCompressionInfo *cminfo) +{ + PQExpBuffer query; + AttributeCompressionItem *item; + int i; + + if (!(cminfo->dobj.dump & DUMP_COMPONENT_DEFINITION)) + return; + + query = createPQExpBuffer(); + resetPQExpBuffer(query); + + for (i = 0; i < cminfo->nitems; i++) + { + item = &cminfo->items[i]; + appendPQExpBuffer(query, "INSERT INTO pg_catalog.pg_attr_compression\n\t" + "VALUES (%d, '%s', '%s'::pg_catalog.regclass, %d, ", + item->acoid, + item->acname, + cminfo->acrelid, + cminfo->acattnum); + if (strlen(item->acoptions) > 0) + appendPQExpBuffer(query, "'%s');\n", item->acoptions); + else + appendPQExpBuffer(query, "NULL);\n"); + + /* + * Restore dependency with access method. pglz is pinned by the + * system, no need to create dependency. + */ + if (strcmp(item->acname, "pglz") != 0) + appendPQExpBuffer(query, + "WITH t AS (SELECT oid FROM pg_am WHERE amname = '%s')\n\t" + "INSERT INTO pg_catalog.pg_depend " + "SELECT %d, %d, 0, %d, t.oid, 0, 'n' FROM t;\n", + item->acname, + AttrCompressionRelationId, + item->acoid, + AccessMethodRelationId); + + if (item->iscurrent) + { + appendPQExpBuffer(query, "ALTER TABLE %s ALTER COLUMN %s\n\tSET COMPRESSION %s", + cminfo->acrelid, + cminfo->attname, + item->acname); + if (strlen(item->parsedoptions) > 0) + appendPQExpBuffer(query, " WITH (%s)", item->parsedoptions); + appendPQExpBuffer(query, " PRESERVE (%s);\n", cminfo->preserve); + } + else if (!IsBuiltinCompression(item->acoid)) + { + /* restore dependency */ + appendPQExpBuffer(query, "INSERT INTO pg_catalog.pg_depend VALUES " + "(%d, %d, 0, %d, '%s'::pg_catalog.regclass, %d, 'i');\n", + AttrCompressionRelationId, + item->acoid, + RelationRelationId, + cminfo->acrelid, + cminfo->acattnum); + } + } + + ArchiveEntry(fout, + cminfo->dobj.catId, + cminfo->dobj.dumpId, + cminfo->dobj.name, + NULL, + NULL, + "", false, + "ATTRIBUTE COMPRESSION", SECTION_PRE_DATA, + query->data, "", NULL, + NULL, 0, + NULL, NULL); + + destroyPQExpBuffer(query); +} + static void binary_upgrade_set_type_oids_by_type_oid(Archive *fout, PQExpBuffer upgrade_buffer, @@ -8068,6 +8304,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) int i_attoptions; int i_attcollation; int i_attfdwoptions; + int i_attcmoptions; + int i_attcmname; PGresult *res; int ntups; bool hasdefaults; @@ -8097,7 +8335,46 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) resetPQExpBuffer(q); - if (fout->remoteVersion >= 100000) + if (fout->remoteVersion >= 110000) + { + /* + * attcompression is new in version 11 + */ + appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " + "a.attstattarget, a.attstorage, t.typstorage, " + "a.attnotnull, a.atthasdef, a.attisdropped, " + "a.attlen, a.attalign, a.attislocal, " + "pg_catalog.format_type(t.oid,a.atttypmod) AS atttypname, " + "array_to_string(a.attoptions, ', ') AS attoptions, " + "CASE WHEN a.attcollation <> t.typcollation " + "THEN a.attcollation ELSE 0 END AS attcollation, " + "a.attidentity, " + /* fdw options */ + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(attfdwoptions) " + "ORDER BY option_name" + "), E',\n ') AS attfdwoptions, " + /* compression */ + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(c.acoptions) " + "ORDER BY option_name" + "), E',\n ') AS attcmoptions, " + "c.acname AS attcmname " + /* FROM */ + "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " + "ON a.atttypid = t.oid " + "LEFT JOIN pg_catalog.pg_attr_compression c " + "ON a.attcompression = c.acoid " + "WHERE a.attrelid = '%u'::pg_catalog.oid " + "AND a.attnum > 0::pg_catalog.int2 " + "ORDER BY a.attnum", + tbinfo->dobj.catId.oid); + } + else if (fout->remoteVersion >= 100000) { /* * attidentity is new in version 10. @@ -8116,7 +8393,9 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "' ' || pg_catalog.quote_literal(option_value) " "FROM pg_catalog.pg_options_to_table(attfdwoptions) " "ORDER BY option_name" - "), E',\n ') AS attfdwoptions " + "), E',\n ') AS attfdwoptions, " + "NULL AS attcmoptions, " + "NULL AS attcmname " "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " @@ -8142,7 +8421,9 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "' ' || pg_catalog.quote_literal(option_value) " "FROM pg_catalog.pg_options_to_table(attfdwoptions) " "ORDER BY option_name" - "), E',\n ') AS attfdwoptions " + "), E',\n ') AS attfdwoptions, " + "NULL AS attcmoptions, " + "NULL AS attcmname " "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " @@ -8166,7 +8447,9 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "array_to_string(a.attoptions, ', ') AS attoptions, " "CASE WHEN a.attcollation <> t.typcollation " "THEN a.attcollation ELSE 0 END AS attcollation, " - "NULL AS attfdwoptions " + "NULL AS attfdwoptions, " + "NULL AS attcmoptions, " + "NULL AS attcmname " "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " @@ -8184,7 +8467,9 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "pg_catalog.format_type(t.oid,a.atttypmod) AS atttypname, " "array_to_string(a.attoptions, ', ') AS attoptions, " "0 AS attcollation, " - "NULL AS attfdwoptions " + "NULL AS attfdwoptions, " + "NULL AS attcmoptions, " + "NULL AS attcmname " "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " @@ -8201,7 +8486,9 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "a.attlen, a.attalign, a.attislocal, " "pg_catalog.format_type(t.oid,a.atttypmod) AS atttypname, " "'' AS attoptions, 0 AS attcollation, " - "NULL AS attfdwoptions " + "NULL AS attfdwoptions, " + "NULL AS attcmoptions, " + "NULL AS attcmname " "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " @@ -8231,6 +8518,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) i_attoptions = PQfnumber(res, "attoptions"); i_attcollation = PQfnumber(res, "attcollation"); i_attfdwoptions = PQfnumber(res, "attfdwoptions"); + i_attcmname = PQfnumber(res, "attcmname"); + i_attcmoptions = PQfnumber(res, "attcmoptions"); tbinfo->numatts = ntups; tbinfo->attnames = (char **) pg_malloc(ntups * sizeof(char *)); @@ -8247,6 +8536,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid)); tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmoptions = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmnames = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *)); @@ -8274,6 +8565,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attoptions[j] = pg_strdup(PQgetvalue(res, j, i_attoptions)); tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, i_attcollation)); tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, i_attfdwoptions)); + tbinfo->attcmoptions[j] = pg_strdup(PQgetvalue(res, j, i_attcmoptions)); + tbinfo->attcmnames[j] = pg_strdup(PQgetvalue(res, j, i_attcmname)); tbinfo->attrdefs[j] = NULL; /* fix below */ if (PQgetvalue(res, j, i_atthasdef)[0] == 't') hasdefaults = true; @@ -9764,6 +10057,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj) case DO_SUBSCRIPTION: dumpSubscription(fout, (SubscriptionInfo *) dobj); break; + case DO_ATTRIBUTE_COMPRESSION: + dumpAttributeCompression(fout, (AttributeCompressionInfo *) dobj); + break; case DO_PRE_DATA_BOUNDARY: case DO_POST_DATA_BOUNDARY: /* never dumped, nothing to do */ @@ -12526,6 +12822,9 @@ dumpAccessMethod(Archive *fout, AccessMethodInfo *aminfo) case AMTYPE_INDEX: appendPQExpBuffer(q, "TYPE INDEX "); break; + case AMTYPE_COMPRESSION: + appendPQExpBuffer(q, "TYPE COMPRESSION "); + break; default: write_msg(NULL, "WARNING: invalid type \"%c\" of access method \"%s\"\n", aminfo->amtype, qamname); @@ -15451,6 +15750,14 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) (!tbinfo->inhNotNull[j] || dopt->binary_upgrade)); + /* + * Compression will require a record in + * pg_attr_compression + */ + bool has_custom_compression = (tbinfo->attcmnames[j] && + ((strcmp(tbinfo->attcmnames[j], "pglz") != 0) || + nonemptyReloptions(tbinfo->attcmoptions[j]))); + /* * Skip column if fully defined by reloftype or the * partition parent. @@ -15509,6 +15816,25 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) fmtQualifiedDumpable(coll)); } + /* + * Compression + * + * In binary-upgrade mode, compression is assigned by + * ALTER. Even we're skipping compression the attribute + * will get builtin compression. It's the task for ALTER + * command to change to right one and remove dependency to + * builtin compression. + */ + if (tbinfo->attcmnames[j] && strlen(tbinfo->attcmnames[j]) + && !(dopt->binary_upgrade && has_custom_compression)) + { + appendPQExpBuffer(q, " COMPRESSION %s", + tbinfo->attcmnames[j]); + if (nonemptyReloptions(tbinfo->attcmoptions[j])) + appendPQExpBuffer(q, " WITH (%s)", + tbinfo->attcmoptions[j]); + } + if (has_default) appendPQExpBuffer(q, " DEFAULT %s", tbinfo->attrdefs[j]->adef_expr); @@ -17769,6 +18095,7 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs, case DO_FOREIGN_SERVER: case DO_TRANSFORM: case DO_BLOB: + case DO_ATTRIBUTE_COMPRESSION: /* Pre-data objects: must come before the pre-data boundary */ addObjectDependency(preDataBound, dobj->dumpId); break; diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index a4d6d926a8..9585040ed6 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -84,7 +84,8 @@ typedef enum DO_POLICY, DO_PUBLICATION, DO_PUBLICATION_REL, - DO_SUBSCRIPTION + DO_SUBSCRIPTION, + DO_ATTRIBUTE_COMPRESSION } DumpableObjectType; /* component types of an object which can be selected for dumping */ @@ -316,6 +317,8 @@ typedef struct _tableInfo char **attoptions; /* per-attribute options */ Oid *attcollation; /* per-attribute collation selection */ char **attfdwoptions; /* per-attribute fdw options */ + char **attcmoptions; /* per-attribute compression options */ + char **attcmnames; /* per-attribute compression method names */ bool *notnull; /* NOT NULL constraints on attributes */ bool *inhNotNull; /* true if NOT NULL is inherited */ struct _attrDefInfo **attrdefs; /* DEFAULT expressions */ @@ -621,6 +624,28 @@ typedef struct _SubscriptionInfo char *subpublications; } SubscriptionInfo; +typedef struct _AttributeCompressionItem +{ + Oid acoid; + char *acname; + char *acoptions; + char *parsedoptions; + bool iscurrent; +} AttributeCompressionItem; + +/* The AttributeCompressionInfo struct is used to represent attribute compression */ +typedef struct _AttributeCompressionInfo +{ + DumpableObject dobj; + char *acrelid; + uint16 acattnum; + char *attname; + char *preserve; + + int nitems; + AttributeCompressionItem *items; +} AttributeCompressionInfo; + /* * We build an array of these with an entry for each object that is an * extension member according to pg_depend. @@ -721,5 +746,6 @@ extern void getPublications(Archive *fout); extern void getPublicationTables(Archive *fout, TableInfo tblinfo[], int numTables); extern void getSubscriptions(Archive *fout); +extern void getAttributeCompressionData(Archive *fout); #endif /* PG_DUMP_H */ diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c index 5ce3c5d485..463afaa3c3 100644 --- a/src/bin/pg_dump/pg_dump_sort.c +++ b/src/bin/pg_dump/pg_dump_sort.c @@ -49,7 +49,7 @@ static const int dbObjectTypePriority[] = 6, /* DO_FUNC */ 7, /* DO_AGG */ 8, /* DO_OPERATOR */ - 8, /* DO_ACCESS_METHOD */ + 7, /* DO_ACCESS_METHOD */ 9, /* DO_OPCLASS */ 9, /* DO_OPFAMILY */ 3, /* DO_COLLATION */ @@ -85,7 +85,8 @@ static const int dbObjectTypePriority[] = 35, /* DO_POLICY */ 36, /* DO_PUBLICATION */ 37, /* DO_PUBLICATION_REL */ - 38 /* DO_SUBSCRIPTION */ + 38, /* DO_SUBSCRIPTION */ + 21 /* DO_ATTRIBUTE_COMPRESSION */ }; static DumpId preDataBoundId; @@ -1470,6 +1471,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize) "POST-DATA BOUNDARY (ID %d)", obj->dumpId); return; + case DO_ATTRIBUTE_COMPRESSION: + snprintf(buf, bufsize, + "ATTRIBUTE COMPRESSION %s (ID %d OID %u)", + obj->name, obj->dumpId, obj->catId.oid); + return; } /* shouldn't get here */ snprintf(buf, bufsize, diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 75e4a539bf..575449b22b 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -74,6 +74,7 @@ static int no_comments = 0; static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; +static int no_compression_methods = 0; static int no_unlogged_table_data = 0; static int no_role_passwords = 0; static int server_version; @@ -135,6 +136,7 @@ main(int argc, char *argv[]) {"no-role-passwords", no_argument, &no_role_passwords, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {"no-sync", no_argument, NULL, 4}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, @@ -404,6 +406,8 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-security-labels"); if (no_subscriptions) appendPQExpBufferStr(pgdumpopts, " --no-subscriptions"); + if (no_compression_methods) + appendPQExpBufferStr(pgdumpopts, " --no-compression-methods"); if (no_unlogged_table_data) appendPQExpBufferStr(pgdumpopts, " --no-unlogged-table-data"); @@ -617,6 +621,7 @@ help(void) printf(_(" --no-role-passwords do not dump passwords for roles\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); + printf(_(" --no-compression-methods do not dump compression methods\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index edc14f176f..33399676aa 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -75,6 +75,7 @@ main(int argc, char **argv) static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; + static int no_compression_methods = 0; static int strict_names = 0; struct option cmdopts[] = { @@ -124,6 +125,7 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {NULL, 0, NULL, 0} }; @@ -364,6 +366,7 @@ main(int argc, char **argv) opts->no_publications = no_publications; opts->no_security_labels = no_security_labels; opts->no_subscriptions = no_subscriptions; + opts->no_compression_methods = no_compression_methods; if (if_exists && !opts->dropSchema) { diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 1bea6ae81d..3cf5e4d836 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -904,6 +904,62 @@ my %tests = ( section_pre_data => 1, section_data => 1, }, }, + # compression data in binary upgrade mode + 'ALTER TABLE test_table_compression ALTER COLUMN ... SET COMPRESSION' => { + all_runs => 1, + catch_all => 'ALTER TABLE ... commands', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text COMPRESSION pglz,\E\n + \s+\Qcol2 text,\E\n + \s+\Qcol3 text,\E\n + \s+\Qcol4 text\E\n + \); + .* + \QINSERT INTO pg_catalog.pg_attr_compression\E\n + \s+\QVALUES (\E\d+\Q, 'pglz2', 'dump_test.test_table_compression'::pg_catalog.regclass, 2, NULL);\E\n + \QWITH t AS (SELECT oid FROM pg_am WHERE amname = 'pglz2')\E\n + \s+\QINSERT INTO pg_catalog.pg_depend SELECT 4001, \E\d+\Q, 0, 2601, t.oid, 0, 'n' FROM t;\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col2\E\n + \s+\QSET COMPRESSION pglz2 PRESERVE (pglz2);\E\n + .* + \QINSERT INTO pg_catalog.pg_attr_compression\E\n + \s+\QVALUES (\E\d+\Q, 'pglz', 'dump_test.test_table_compression'::pg_catalog.regclass, 3, '{min_input_size=1000}');\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col3\E\n + \s+\QSET COMPRESSION pglz WITH (min_input_size '1000') PRESERVE (pglz);\E\n + .* + \QINSERT INTO pg_catalog.pg_attr_compression\E\n + \s+\QVALUES (\E\d+\Q, 'pglz2', 'dump_test.test_table_compression'::pg_catalog.regclass, 4, '{min_input_size=1000}');\E\n + \QWITH t AS (SELECT oid FROM pg_am WHERE amname = 'pglz2')\E\n + \s+\QINSERT INTO pg_catalog.pg_depend SELECT 4001, \E\d+\Q, 0, 2601, t.oid, 0, 'n' FROM t;\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col4\E\n + \s+\QSET COMPRESSION pglz2 WITH (min_input_size '1000') PRESERVE (pglz2);\E\n + /xms, + like => { + binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + exclude_test_table_data => 1, + no_blobs => 1, + no_privs => 1, + no_owner => 1, + only_dump_test_schema => 1, + only_dump_test_table => 1, + pg_dumpall_dbprivs => 1, + schema_only => 1, + section_post_data => 1, + test_schema_plus_blobs => 1, + with_oids => 1, + data_only => 1, + exclude_dump_test_schema => 1, + exclude_test_table => 1, + role => 1, + section_pre_data => 1, + section_data => 1, }, }, + 'ALTER TABLE ONLY test_table ALTER COLUMN col1 SET STATISTICS 90' => { all_runs => 1, catch_all => 'ALTER TABLE ... commands', @@ -2615,6 +2671,39 @@ qr/^\QINSERT INTO dump_test.test_table_identity (col1, col2) OVERRIDING SYSTEM V section_post_data => 1, test_schema_plus_blobs => 1, }, }, + 'CREATE ACCESS METHOD pglz2' => { + all_runs => 1, + catch_all => 'CREATE ... commands', + create_order => 52, + create_sql => + 'CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;', + regexp => + qr/CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;/m, + like => { + binary_upgrade => 1, + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + exclude_dump_test_schema => 1, + exclude_test_table => 1, + exclude_test_table_data => 1, + no_blobs => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_dbprivs => 1, + schema_only => 1, + section_pre_data => 1, + with_oids => 1, }, + unlike => { + only_dump_test_schema => 1, + only_dump_test_table => 1, + pg_dumpall_globals => 1, + pg_dumpall_globals_clean => 1, + role => 1, + section_post_data => 1, + test_schema_plus_blobs => 1, }, }, + 'CREATE COLLATION test0 FROM "C"' => { all_runs => 1, catch_all => 'CREATE ... commands', @@ -4669,9 +4758,9 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog regexp => qr/^ \QCREATE TABLE dump_test.test_table (\E\n \s+\Qcol1 integer NOT NULL,\E\n - \s+\Qcol2 text,\E\n - \s+\Qcol3 text,\E\n - \s+\Qcol4 text,\E\n + \s+\Qcol2 text COMPRESSION pglz,\E\n + \s+\Qcol3 text COMPRESSION pglz,\E\n + \s+\Qcol4 text COMPRESSION pglz,\E\n \s+\QCONSTRAINT test_table_col1_check CHECK ((col1 <= 1000))\E\n \Q)\E\n \QWITH (autovacuum_enabled='false', fillfactor='80');\E\n/xm, @@ -4748,7 +4837,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog regexp => qr/^ \QCREATE TABLE dump_test.test_second_table (\E \n\s+\Qcol1 integer,\E - \n\s+\Qcol2 text\E + \n\s+\Qcol2 text COMPRESSION pglz\E \n\); /xm, like => { @@ -4955,7 +5044,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog \n\s+\Qcol1 integer,\E \n\s+\Qcol2 boolean,\E \n\s+\Qcol3 boolean,\E - \n\s+\Qcol4 bit(5),\E + \n\s+\Qcol4 bit(5) COMPRESSION pglz,\E \n\s+\Qcol5 double precision\E \n\); /xm, @@ -4995,7 +5084,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog regexp => qr/^ \QCREATE TABLE dump_test.test_table_identity (\E\n \s+\Qcol1 integer NOT NULL,\E\n - \s+\Qcol2 text\E\n + \s+\Qcol2 text COMPRESSION pglz\E\n \); .* \QALTER TABLE dump_test.test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n @@ -5032,6 +5121,49 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog role => 1, section_post_data => 1, }, }, + 'CREATE TABLE test_table_compression' => { + all_runs => 1, + catch_all => 'CREATE ... commands', + create_order => 53, + create_sql => 'CREATE TABLE dump_test.test_table_compression ( + col1 text, + col2 text COMPRESSION pglz2, + col3 text COMPRESSION pglz WITH (min_input_size \'1000\'), + col4 text COMPRESSION pglz2 WITH (min_input_size \'1000\') + );', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text COMPRESSION pglz,\E\n + \s+\Qcol2 text COMPRESSION pglz2,\E\n + \s+\Qcol3 text COMPRESSION pglz WITH (min_input_size '1000'),\E\n + \s+\Qcol4 text COMPRESSION pglz2 WITH (min_input_size '1000')\E\n + \); + /xm, + like => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + exclude_test_table => 1, + exclude_test_table_data => 1, + no_blobs => 1, + no_privs => 1, + no_owner => 1, + only_dump_test_schema => 1, + pg_dumpall_dbprivs => 1, + schema_only => 1, + section_pre_data => 1, + test_schema_plus_blobs => 1, + with_oids => 1, }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + only_dump_test_table => 1, + pg_dumpall_globals => 1, + pg_dumpall_globals_clean => 1, + role => 1, + section_post_data => 1, }, }, + 'CREATE STATISTICS extended_stats_no_options' => { all_runs => 1, catch_all => 'CREATE ... commands', diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 0c3be1f504..7bd0caf2f6 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1755,6 +1755,22 @@ describeOneTableDetails(const char *schemaname, if (verbose) { appendPQExpBufferStr(&buf, ",\n a.attstorage"); + + /* compresssion info */ + if (pset.sversion >= 110000) + appendPQExpBufferStr(&buf, ",\n CASE WHEN attcompression = 0 THEN NULL ELSE " + " (SELECT c.acname || " + " (CASE WHEN acoptions IS NULL " + " THEN '' " + " ELSE '(' || array_to_string(ARRAY(SELECT quote_ident(option_name) || ' ' || quote_literal(option_value)" + " FROM pg_options_to_table(acoptions)), ', ') || ')'" + " END) " + " FROM pg_catalog.pg_attr_compression c " + " WHERE c.acoid = a.attcompression) " + " END AS attcmname"); + else + appendPQExpBufferStr(&buf, "\n NULL AS attcmname"); + appendPQExpBufferStr(&buf, ",\n CASE WHEN a.attstattarget=-1 THEN NULL ELSE a.attstattarget END AS attstattarget"); /* @@ -1871,6 +1887,11 @@ describeOneTableDetails(const char *schemaname, if (verbose) { headers[cols++] = gettext_noop("Storage"); + + if (tableinfo.relkind == RELKIND_RELATION || + tableinfo.relkind == RELKIND_PARTITIONED_TABLE) + headers[cols++] = gettext_noop("Compression"); + if (tableinfo.relkind == RELKIND_RELATION || tableinfo.relkind == RELKIND_INDEX || tableinfo.relkind == RELKIND_PARTITIONED_INDEX || @@ -1968,6 +1989,12 @@ describeOneTableDetails(const char *schemaname, "???")))), false, false); + /* Column compression. */ + if (tableinfo.relkind == RELKIND_RELATION || + tableinfo.relkind == RELKIND_PARTITIONED_TABLE) + printTableAddCell(&cont, PQgetvalue(res, i, firstvcol + 1), + false, false); + /* Statistics target, if the relkind supports this feature */ if (tableinfo.relkind == RELKIND_RELATION || tableinfo.relkind == RELKIND_INDEX || @@ -1976,7 +2003,7 @@ describeOneTableDetails(const char *schemaname, tableinfo.relkind == RELKIND_FOREIGN_TABLE || tableinfo.relkind == RELKIND_PARTITIONED_TABLE) { - printTableAddCell(&cont, PQgetvalue(res, i, firstvcol + 1), + printTableAddCell(&cont, PQgetvalue(res, i, firstvcol + 2), false, false); } @@ -1987,7 +2014,7 @@ describeOneTableDetails(const char *schemaname, tableinfo.relkind == RELKIND_COMPOSITE_TYPE || tableinfo.relkind == RELKIND_FOREIGN_TABLE || tableinfo.relkind == RELKIND_PARTITIONED_TABLE) - printTableAddCell(&cont, PQgetvalue(res, i, firstvcol + 2), + printTableAddCell(&cont, PQgetvalue(res, i, firstvcol + 3), false, false); } } diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 08d8ef09a4..46af83344f 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2159,11 +2159,14 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] SET */ else if (Matches7("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET") || Matches6("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET")) - COMPLETE_WITH_LIST5("(", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); + COMPLETE_WITH_LIST6("(", "COMPRESSION", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); /* ALTER TABLE ALTER [COLUMN] SET ( */ else if (Matches8("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") || Matches7("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "(")) COMPLETE_WITH_LIST2("n_distinct", "n_distinct_inherited"); + else if (Matches9("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION", MatchAny) || + Matches8("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION", MatchAny)) + COMPLETE_WITH_CONST("WITH ("); /* ALTER TABLE ALTER [COLUMN] SET STORAGE */ else if (Matches8("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "STORAGE") || Matches7("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "STORAGE")) diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h new file mode 100644 index 0000000000..4d21d4303e --- /dev/null +++ b/src/include/access/cmapi.h @@ -0,0 +1,86 @@ +/*------------------------------------------------------------------------- + * + * cmapi.h + * API for Postgres compression AM. + * + * Copyright (c) 2015-2017, PostgreSQL Global Development Group + * + * src/include/access/cmapi.h + * + *------------------------------------------------------------------------- + */ + +#ifndef CMAPI_H +#define CMAPI_H + +#include "postgres.h" +#include "access/transam.h" +#include "catalog/pg_attr_compression.h" +#include "catalog/pg_attribute.h" +#include "nodes/pg_list.h" + +#define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) +#define DefaultCompressionOid (PGLZ_AC_OID) + +typedef struct CompressionAmRoutine CompressionAmRoutine; + +/* + * CompressionAmOptions contains all information needed to compress varlena. + * + * For optimization purposes it will be created once for each attribute + * compression and stored in cache, until its renewal on global cache reset, + * or until deletion of related attribute compression. + */ +typedef struct CompressionAmOptions +{ + Oid acoid; /* Oid of attribute compression */ + Oid amoid; /* Oid of compression access method */ + List *acoptions; /* Parsed options, used for comparison */ + CompressionAmRoutine *amroutine; /* compression access method routine */ + + /* result of cminitstate function will be put here */ + void *acstate; +} CompressionAmOptions; + +typedef void (*cmcheck_function) (Form_pg_attribute att, List *options); +typedef void (*cmdrop_function) (Oid acoid); +typedef struct varlena *(*cmcompress_function) + (CompressionAmOptions *cmoptions, const struct varlena *value); +typedef void *(*cminitstate_function) (Oid acoid, List *options); + +/* + * API struct for a compression AM. + * + * 'cmcheck' - called when attribute is linking with compression method. + * This function should check compability of compression method with + * the attribute and its options. + * + * 'cmdrop' - called before drop of attribute compression. Could be used + * by an extension to cleanup some data related with attribute compression + * (like dictionaries etc). + * + * 'cminitstate' - called when CompressionAmOptions instance is created. + * Should return pointer to a memory in a caller memory context, or NULL. + * Could be used to pass some internal state between compression function + * calls, like internal structure for parsed compression options. + * + * 'cmcompress' and 'cmdecompress' - varlena compression functions. + */ +typedef struct CompressionAmRoutine +{ + NodeTag type; + + cmcheck_function cmcheck; /* can be NULL */ + cmdrop_function cmdrop; /* can be NULL */ + cminitstate_function cminitstate; /* can be NULL */ + cmcompress_function cmcompress; + cmcompress_function cmdecompress; +} CompressionAmRoutine; + +/* access/compression/cmapi.c */ +extern CompressionAmRoutine *InvokeCompressionAmHandler(Oid amhandler); +extern CompressionAmRoutine *GetCompressionAmRoutine(Oid acoid); +extern List *GetAttrCompressionOptions(Oid acoid); +extern Oid GetAttrCompressionAmOid(Oid acoid); + +#endif /* CMAPI_H */ diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 4c0256b18a..9b091ff583 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -47,6 +47,9 @@ typedef enum LockTupleMode LockTupleExclusive } LockTupleMode; +/* Hash table control struct is an opaque type known only within dynahash.c */ +typedef struct HTAB HTAB; + #define MaxLockTupleMode LockTupleExclusive /* @@ -72,7 +75,6 @@ typedef struct HeapUpdateFailureData CommandId cmax; } HeapUpdateFailureData; - /* ---------------- * function prototypes for heap access method * @@ -146,7 +148,7 @@ extern void heap_get_latest_tid(Relation relation, Snapshot snapshot, ItemPointer tid); extern void setLastTid(const ItemPointer tid); -extern BulkInsertState GetBulkInsertState(void); +extern BulkInsertState GetBulkInsertState(HTAB *); extern void FreeBulkInsertState(BulkInsertState); extern void ReleaseBulkInsertStatePin(BulkInsertState bistate); diff --git a/src/include/access/hio.h b/src/include/access/hio.h index 9993d5be70..b491bf6338 100644 --- a/src/include/access/hio.h +++ b/src/include/access/hio.h @@ -32,6 +32,8 @@ typedef struct BulkInsertStateData { BufferAccessStrategy strategy; /* our BULKWRITE strategy object */ Buffer current_buf; /* current insertion target page */ + HTAB *preserved_am_info; /* hash table with preserved compression + * methods for attributes */ } BulkInsertStateData; diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 3616a17b6f..0d59a20b76 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -269,7 +269,9 @@ struct HeapTupleHeaderData * information stored in t_infomask2: */ #define HEAP_NATTS_MASK 0x07FF /* 11 bits for number of attributes */ -/* bits 0x1800 are available */ +/* bit 0x800 is available */ +#define HEAP_HASCUSTOMCOMPRESSED 0x1000 /* tuple contains custom compressed + * varlena(s) */ #define HEAP_KEYS_UPDATED 0x2000 /* tuple was updated and key cols * modified, or tuple deleted */ #define HEAP_HOT_UPDATED 0x4000 /* tuple was HOT-updated */ @@ -683,6 +685,9 @@ struct MinimalTupleData #define HeapTupleHasExternal(tuple) \ (((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0) +#define HeapTupleHasCustomCompressed(tuple) \ + (((tuple)->t_data->t_infomask2 & HEAP_HASCUSTOMCOMPRESSED) != 0) + #define HeapTupleIsHotUpdated(tuple) \ HeapTupleHeaderIsHotUpdated((tuple)->t_data) @@ -798,7 +803,7 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc, extern void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, char *data, Size data_size, - uint16 *infomask, bits8 *bit); + uint16 *infomask, uint16 *infomask2, bits8 *bit); extern bool heap_attisnull(HeapTuple tup, int attnum); extern Datum nocachegetattr(HeapTuple tup, int attnum, TupleDesc att); diff --git a/src/include/access/reloptions.h b/src/include/access/reloptions.h index b32c1e9efe..fb088f08e7 100644 --- a/src/include/access/reloptions.h +++ b/src/include/access/reloptions.h @@ -258,7 +258,6 @@ extern void add_string_reloption(bits32 kinds, const char *name, const char *des extern Datum transformRelOptions(Datum oldOptions, List *defList, const char *namspace, char *validnsps[], bool ignoreOids, bool isReset); -extern List *untransformRelOptions(Datum options); extern bytea *extractRelOptions(HeapTuple tuple, TupleDesc tupdesc, amoptions_function amoptions); extern relopt_value *parseRelOptions(Datum options, bool validate, @@ -269,6 +268,8 @@ extern void fillRelOptions(void *rdopts, Size basesize, relopt_value *options, int numoptions, bool validate, const relopt_parse_elt *elems, int nelems); +extern char *formatRelOptions(List *options); +extern List *untransformRelOptions(Datum options); extern bytea *default_reloptions(Datum reloptions, bool validate, relopt_kind kind); diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h index 415efbab97..2747a9bc55 100644 --- a/src/include/access/tupdesc.h +++ b/src/include/access/tupdesc.h @@ -14,7 +14,9 @@ #ifndef TUPDESC_H #define TUPDESC_H +#include "postgres.h" #include "access/attnum.h" +#include "access/cmapi.h" #include "catalog/pg_attribute.h" #include "nodes/pg_list.h" @@ -43,6 +45,10 @@ typedef struct tupleConstr bool has_not_null; } TupleConstr; +/* tupledesc flags */ +#define TD_ATTR_CUSTOM_COMPRESSED 0x01 /* is TupleDesc contain attributes + * with custom compression? */ + /* * This struct is passed around within the backend to describe the structure * of tuples. For tuples coming from on-disk relations, the information is @@ -80,6 +86,7 @@ typedef struct tupleDesc Oid tdtypeid; /* composite type ID for tuple type */ int32 tdtypmod; /* typmod for tuple type */ bool tdhasoid; /* tuple has oid attribute in its header */ + char tdflags; /* tuple additional flags */ int tdrefcount; /* reference count, or -1 if not counting */ TupleConstr *constr; /* constraints, or NULL if none */ /* attrs[N] is the description of Attribute Number N+1 */ diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index f99291e30d..d5561406cc 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -101,6 +101,15 @@ /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) +/* + * va_extinfo in varatt_external contains actual length of the external data + * and optional flags + */ +#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ + ((toast_pointer).va_extinfo & 0x3FFFFFFF) +#define VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer) \ + (((toast_pointer).va_extinfo >> 30) == 0x02) + /* * Testing whether an externally-stored value is compressed now requires * comparing extsize (the actual length of the external data) to rawsize @@ -109,7 +118,7 @@ * saves space, so we expect either equality or less-than. */ #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ - ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ) + (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < (toast_pointer).va_rawsize - VARHDRSZ) /* * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum @@ -126,6 +135,19 @@ do { \ memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \ } while (0) +/* Hash table control struct is an opaque type known only within dynahash.c */ +typedef struct HTAB HTAB; + +/* + * Used to pass to preserved compression access methods from + * ALTER TABLE SET COMPRESSION into toast_insert_or_update. + */ +typedef struct AttrCmPreservedInfo +{ + AttrNumber attnum; + List *preserved_amoids; +} AttrCmPreservedInfo; + /* ---------- * toast_insert_or_update - * @@ -134,7 +156,7 @@ do { \ */ extern HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, - int options); + int options, HTAB *preserved_cmlist); /* ---------- * toast_delete - @@ -210,7 +232,7 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, * Create a compressed version of a varlena datum, if possible * ---------- */ -extern Datum toast_compress_datum(Datum value); +extern Datum toast_compress_datum(Datum value, Oid cmoptoid); /* ---------- * toast_raw_datum_size - diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 46c271a46c..cb19cb3e41 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -180,10 +180,11 @@ typedef enum ObjectClass OCLASS_PUBLICATION, /* pg_publication */ OCLASS_PUBLICATION_REL, /* pg_publication_rel */ OCLASS_SUBSCRIPTION, /* pg_subscription */ - OCLASS_TRANSFORM /* pg_transform */ + OCLASS_TRANSFORM, /* pg_transform */ + OCLASS_ATTR_COMPRESSION /* pg_attr_compression */ } ObjectClass; -#define LAST_OCLASS OCLASS_TRANSFORM +#define LAST_OCLASS OCLASS_ATTR_COMPRESSION /* flag bits for performDeletion/performMultipleDeletions: */ #define PERFORM_DELETION_INTERNAL 0x0001 /* internal action */ diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index 7dd9d108d6..1eb3e319c5 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -88,6 +88,11 @@ DECLARE_UNIQUE_INDEX(pg_attrdef_adrelid_adnum_index, 2656, on pg_attrdef using b DECLARE_UNIQUE_INDEX(pg_attrdef_oid_index, 2657, on pg_attrdef using btree(oid oid_ops)); #define AttrDefaultOidIndexId 2657 +DECLARE_UNIQUE_INDEX(pg_attr_compression_index, 4003, on pg_attr_compression using btree(acoid oid_ops)); +#define AttrCompressionIndexId 4003 +DECLARE_INDEX(pg_attr_compression_relid_attnum_index, 4004, on pg_attr_compression using btree(acrelid oid_ops, acattnum int2_ops)); +#define AttrCompressionRelidAttnumIndexId 4004 + DECLARE_UNIQUE_INDEX(pg_attribute_relid_attnam_index, 2658, on pg_attribute using btree(attrelid oid_ops, attname name_ops)); #define AttributeRelidNameIndexId 2658 DECLARE_UNIQUE_INDEX(pg_attribute_relid_attnum_index, 2659, on pg_attribute using btree(attrelid oid_ops, attnum int2_ops)); diff --git a/src/include/catalog/pg_am.h b/src/include/catalog/pg_am.h index 2e785c4cec..5cb2852f41 100644 --- a/src/include/catalog/pg_am.h +++ b/src/include/catalog/pg_am.h @@ -59,6 +59,7 @@ typedef FormData_pg_am *Form_pg_am; * ---------------- */ #define AMTYPE_INDEX 'i' /* index access method */ +#define AMTYPE_COMPRESSION 'c' /* compression access method */ /* ---------------- * initial contents of pg_am @@ -83,5 +84,8 @@ DESCR("SP-GiST index access method"); DATA(insert OID = 3580 ( brin brinhandler i )); DESCR("block range index (BRIN) access method"); #define BRIN_AM_OID 3580 +DATA(insert OID = 4002 ( pglz pglzhandler c )); +DESCR("PGLZ compression access method"); +#define PGLZ_COMPRESSION_AM_OID 4002 #endif /* PG_AM_H */ diff --git a/src/include/catalog/pg_attr_compression.h b/src/include/catalog/pg_attr_compression.h new file mode 100644 index 0000000000..47e739622c --- /dev/null +++ b/src/include/catalog/pg_attr_compression.h @@ -0,0 +1,72 @@ +/*------------------------------------------------------------------------- + * + * pg_attr_compression.h + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_attr_compression.h + * + * NOTES + * the genbki.pl script reads this file and generates .bki + * information from the DATA() statements. + * + * XXX do NOT break up DATA() statements into multiple lines! + * the scripts are not as smart as you might think... + * + *------------------------------------------------------------------------- + */ +#ifndef PG_ATTR_COMPRESSION_H +#define PG_ATTR_COMPRESSION_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_attr_compression definition. cpp turns this into + * typedef struct FormData_pg_attr_compression + * ---------------- + */ +#define AttrCompressionRelationId 4001 + +CATALOG(pg_attr_compression,4001) BKI_WITHOUT_OIDS +{ + Oid acoid; /* attribute compression oid */ + NameData acname; /* name of compression AM */ + Oid acrelid; /* attribute relation */ + int16 acattnum; /* attribute number in the relation */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text acoptions[1]; /* specific options from WITH */ +#endif +} FormData_pg_attr_compression; + +/* ---------------- + * Form_pg_attr_compresssion corresponds to a pointer to a tuple with + * the format of pg_attr_compression relation. + * ---------------- + */ +typedef FormData_pg_attr_compression * Form_pg_attr_compression; + +/* ---------------- + * compiler constants for pg_attr_compression + * ---------------- + */ +#define Natts_pg_attr_compression 5 +#define Anum_pg_attr_compression_acoid 1 +#define Anum_pg_attr_compression_acname 2 +#define Anum_pg_attr_compression_acrelid 3 +#define Anum_pg_attr_compression_acattnum 4 +#define Anum_pg_attr_compression_acoptions 5 + +/* + * Predefined compression options for builtin compression access methods + * It is safe to use Oids that equal to Oids of access methods, since + * these are just numbers and they are not using system Oid column. + * + * Note that predefined options should have 0 in acrelid and acattnum. + */ + +DATA(insert ( 4002 pglz 0 0 _null_ )); +#define PGLZ_AC_OID 4002 + +#endif /* PG_ATTR_COMPRESSION_H */ diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index 8159383834..e63eaa52f2 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -156,6 +156,9 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK /* attribute's collation */ Oid attcollation; + /* attribute's compression options or InvalidOid */ + Oid attcompression BKI_DEFAULT(0); + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* NOTE: The following fields are not present in tuple descriptors. */ @@ -174,10 +177,10 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK * ATTRIBUTE_FIXED_PART_SIZE is the size of the fixed-layout, * guaranteed-not-null part of a pg_attribute row. This is in fact as much * of the row as gets copied into tuple descriptors, so don't expect you - * can access fields beyond attcollation except in a real tuple! + * can access fields beyond attcompression except in a real tuple! */ #define ATTRIBUTE_FIXED_PART_SIZE \ - (offsetof(FormData_pg_attribute,attcollation) + sizeof(Oid)) + (offsetof(FormData_pg_attribute,attcompression) + sizeof(Oid)) /* ---------------- * Form_pg_attribute corresponds to a pointer to a tuple with @@ -191,29 +194,30 @@ typedef FormData_pg_attribute *Form_pg_attribute; * ---------------- */ -#define Natts_pg_attribute 22 -#define Anum_pg_attribute_attrelid 1 -#define Anum_pg_attribute_attname 2 -#define Anum_pg_attribute_atttypid 3 -#define Anum_pg_attribute_attstattarget 4 -#define Anum_pg_attribute_attlen 5 -#define Anum_pg_attribute_attnum 6 -#define Anum_pg_attribute_attndims 7 -#define Anum_pg_attribute_attcacheoff 8 -#define Anum_pg_attribute_atttypmod 9 -#define Anum_pg_attribute_attbyval 10 -#define Anum_pg_attribute_attstorage 11 -#define Anum_pg_attribute_attalign 12 -#define Anum_pg_attribute_attnotnull 13 -#define Anum_pg_attribute_atthasdef 14 -#define Anum_pg_attribute_attidentity 15 -#define Anum_pg_attribute_attisdropped 16 -#define Anum_pg_attribute_attislocal 17 -#define Anum_pg_attribute_attinhcount 18 -#define Anum_pg_attribute_attcollation 19 -#define Anum_pg_attribute_attacl 20 -#define Anum_pg_attribute_attoptions 21 -#define Anum_pg_attribute_attfdwoptions 22 +#define Natts_pg_attribute 23 +#define Anum_pg_attribute_attrelid 1 +#define Anum_pg_attribute_attname 2 +#define Anum_pg_attribute_atttypid 3 +#define Anum_pg_attribute_attstattarget 4 +#define Anum_pg_attribute_attlen 5 +#define Anum_pg_attribute_attnum 6 +#define Anum_pg_attribute_attndims 7 +#define Anum_pg_attribute_attcacheoff 8 +#define Anum_pg_attribute_atttypmod 9 +#define Anum_pg_attribute_attbyval 10 +#define Anum_pg_attribute_attstorage 11 +#define Anum_pg_attribute_attalign 12 +#define Anum_pg_attribute_attnotnull 13 +#define Anum_pg_attribute_atthasdef 14 +#define Anum_pg_attribute_attidentity 15 +#define Anum_pg_attribute_attisdropped 16 +#define Anum_pg_attribute_attislocal 17 +#define Anum_pg_attribute_attinhcount 18 +#define Anum_pg_attribute_attcollation 19 +#define Anum_pg_attribute_attcompression 20 +#define Anum_pg_attribute_attacl 21 +#define Anum_pg_attribute_attoptions 22 +#define Anum_pg_attribute_attfdwoptions 23 /* ---------------- diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 85cdb99f1f..a25daf18c3 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -149,7 +149,7 @@ typedef FormData_pg_class *Form_pg_class; */ DATA(insert OID = 1247 ( pg_type PGNSP 71 0 PGUID 0 0 0 0 0 0 0 f f p r 30 0 t f f f f f t n f 0 3 1 _null_ _null_ _null_)); DESCR(""); -DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 22 0 f f f f f f t n f 0 3 1 _null_ _null_ _null_)); +DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 23 0 f f f f f f t n f 0 3 1 _null_ _null_ _null_)); DESCR(""); DATA(insert OID = 1255 ( pg_proc PGNSP 81 0 PGUID 0 0 0 0 0 0 0 f f p r 28 0 t f f f f f t n f 0 3 1 _null_ _null_ _null_)); DESCR(""); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index bfc90098f8..96df396aac 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -576,6 +576,10 @@ DESCR("brin: standalone scan new table pages"); DATA(insert OID = 4014 ( brin_desummarize_range PGNSP PGUID 12 1 0 0 0 f f f t f v s 2 0 2278 "2205 20" _null_ _null_ _null_ _null_ _null_ brin_desummarize_range _null_ _null_ _null_ )); DESCR("brin: desummarize page range"); +/* Compression access method handlers */ +DATA(insert OID = 3453 ( pglzhandler PGNSP PGUID 12 1 0 0 0 f f f t f v s 1 0 4005 "2281" _null_ _null_ _null_ _null_ _null_ pglzhandler _null_ _null_ _null_ )); +DESCR("pglz compression access method handler"); + DATA(insert OID = 338 ( amvalidate PGNSP PGUID 12 1 0 0 0 f f f t f v s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ amvalidate _null_ _null_ _null_ )); DESCR("validate an operator class"); @@ -3843,6 +3847,8 @@ DATA(insert OID = 3454 ( pg_filenode_relation PGNSP PGUID 12 1 0 0 0 f f f t f s DESCR("relation OID for filenode and tablespace"); DATA(insert OID = 3034 ( pg_relation_filepath PGNSP PGUID 12 1 0 0 0 f f f t f s s 1 0 25 "2205" _null_ _null_ _null_ _null_ _null_ pg_relation_filepath _null_ _null_ _null_ )); DESCR("file path of relation"); +DATA(insert OID = 4008 ( pg_column_compression PGNSP PGUID 12 1 0 0 0 f f f t f s s 2 0 25 "2205 25" _null_ _null_ _null_ _null_ _null_ pg_column_compression _null_ _null_ _null_ )); +DESCR("list of compression access methods used by the column"); DATA(insert OID = 2316 ( postgresql_fdw_validator PGNSP PGUID 12 1 0 0 0 f f f t f i s 2 0 16 "1009 26" _null_ _null_ _null_ _null_ _null_ postgresql_fdw_validator _null_ _null_ _null_)); DESCR("(internal)"); @@ -3915,6 +3921,10 @@ DATA(insert OID = 3311 ( tsm_handler_in PGNSP PGUID 12 1 0 0 0 f f f f f i s 1 DESCR("I/O"); DATA(insert OID = 3312 ( tsm_handler_out PGNSP PGUID 12 1 0 0 0 f f f t f i s 1 0 2275 "3310" _null_ _null_ _null_ _null_ _null_ tsm_handler_out _null_ _null_ _null_ )); DESCR("I/O"); +DATA(insert OID = 4006 ( compression_am_handler_in PGNSP PGUID 12 1 0 0 0 f f f f f i s 1 0 4005 "2275" _null_ _null_ _null_ _null_ _null_ compression_am_handler_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 4007 ( compression_am_handler_out PGNSP PGUID 12 1 0 0 0 f f f t f i s 1 0 2275 "4005" _null_ _null_ _null_ _null_ _null_ compression_am_handler_out _null_ _null_ _null_ )); +DESCR("I/O"); /* tablesample method handlers */ DATA(insert OID = 3313 ( bernoulli PGNSP PGUID 12 1 0 0 0 f f f t f v s 1 0 3310 "2281" _null_ _null_ _null_ _null_ _null_ tsm_bernoulli_handler _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 5b5b1218de..1b0283122b 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -712,6 +712,8 @@ DATA(insert OID = 3310 ( tsm_handler PGNSP PGUID 4 t p P f t \054 0 0 0 tsm_han #define TSM_HANDLEROID 3310 DATA(insert OID = 3831 ( anyrange PGNSP PGUID -1 f p P f t \054 0 0 0 anyrange_in anyrange_out - - - - - d x f 0 -1 0 0 _null_ _null_ _null_ )); #define ANYRANGEOID 3831 +DATA(insert OID = 4005 ( compression_am_handler PGNSP PGUID 4 t p P f t \054 0 0 0 compression_am_handler_in compression_am_handler_out - - - - - i p f 0 -1 0 0 _null_ _null_ _null_ )); +#define COMPRESSION_AM_HANDLEROID 4005 /* diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 8fc9e424cf..9acbebb0bc 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -145,6 +145,7 @@ extern Oid RemoveUserMapping(DropUserMappingStmt *stmt); extern void RemoveUserMappingById(Oid umId); extern void CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid); extern void ImportForeignSchema(ImportForeignSchemaStmt *stmt); +extern Datum optionListToArray(List *options, bool sorted); extern Datum transformGenericOptions(Oid catalogId, Datum oldOptions, List *options, @@ -154,8 +155,21 @@ extern Datum transformGenericOptions(Oid catalogId, extern ObjectAddress CreateAccessMethod(CreateAmStmt *stmt); extern void RemoveAccessMethodById(Oid amOid); extern Oid get_index_am_oid(const char *amname, bool missing_ok); +extern Oid get_compression_am_oid(const char *amname, bool missing_ok); extern Oid get_am_oid(const char *amname, bool missing_ok); extern char *get_am_name(Oid amOid); +extern regproc get_am_handler_oid(Oid amOid, char amtype, bool noerror); + +/* commands/compressioncmds.c */ +extern ColumnCompression *MakeColumnCompression(Oid acoid); +extern Oid CreateAttributeCompression(Form_pg_attribute attr, + ColumnCompression *compression, + bool *need_rewrite, + List **preserved_amoids); +extern void RemoveAttributeCompression(Oid acoid); +extern void CheckCompressionMismatch(ColumnCompression *c1, + ColumnCompression *c2, const char *attributeName); +void CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids); /* support routines in commands/define.c */ diff --git a/src/include/commands/event_trigger.h b/src/include/commands/event_trigger.h index 0e1959462e..4fc2ed8c68 100644 --- a/src/include/commands/event_trigger.h +++ b/src/include/commands/event_trigger.h @@ -32,6 +32,7 @@ typedef struct EventTriggerData #define AT_REWRITE_DEFAULT_VAL 0x02 #define AT_REWRITE_COLUMN_REWRITE 0x04 #define AT_REWRITE_ALTER_OID 0x08 +#define AT_REWRITE_ALTER_COMPRESSION 0x10 /* * EventTriggerData is the node type that is passed as fmgr "context" info diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 443de22704..3edf3bba81 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -471,6 +471,7 @@ typedef enum NodeTag T_PartitionBoundSpec, T_PartitionRangeDatum, T_PartitionCmd, + T_ColumnCompression, T_VacuumRelation, /* @@ -502,7 +503,8 @@ typedef enum NodeTag T_IndexAmRoutine, /* in access/amapi.h */ T_TsmRoutine, /* in access/tsmapi.h */ T_ForeignKeyCacheInfo, /* in utils/rel.h */ - T_CallContext /* in nodes/parsenodes.h */ + T_CallContext, /* in nodes/parsenodes.h */ + T_CompressionAmRoutine /* in access/cmapi.h */ } NodeTag; /* diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 92082b3a7a..72ac1ca3f2 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -621,6 +621,20 @@ typedef struct RangeTableSample int location; /* method name location, or -1 if unknown */ } RangeTableSample; +/* + * ColumnCompression - compression parameters for some attribute + * + * This represents compression information defined using clause: + * .. COMPRESSION WITH () PRESERVE + */ +typedef struct ColumnCompression +{ + NodeTag type; + char *amname; + List *options; + List *preserve; +} ColumnCompression; + /* * ColumnDef - column definition (used in various creates) * @@ -644,6 +658,7 @@ typedef struct ColumnDef NodeTag type; char *colname; /* name of column */ TypeName *typeName; /* type of column */ + ColumnCompression *compression; int inhcount; /* number of times column is inherited */ bool is_local; /* column has local (non-inherited) def'n */ bool is_not_null; /* NOT NULL constraint specified? */ @@ -681,6 +696,7 @@ typedef enum TableLikeOption CREATE_TABLE_LIKE_INDEXES = 1 << 4, CREATE_TABLE_LIKE_STATISTICS = 1 << 5, CREATE_TABLE_LIKE_STORAGE = 1 << 6, + CREATE_TABLE_LIKE_COMPRESSION = 1 << 7, CREATE_TABLE_LIKE_ALL = PG_INT32_MAX } TableLikeOption; @@ -1788,7 +1804,8 @@ typedef enum AlterTableType AT_DetachPartition, /* DETACH PARTITION */ AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ - AT_DropIdentity /* DROP IDENTITY */ + AT_DropIdentity, /* DROP IDENTITY */ + AT_SetCompression /* SET COMPRESSION */ } AlterTableType; typedef struct ReplicaIdentityStmt diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index cf32197bc3..773d896b9b 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD) PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD) PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD) +PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD) PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD) PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD) diff --git a/src/include/postgres.h b/src/include/postgres.h index bbcb50e41f..abba43cb94 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -55,7 +55,7 @@ /* * struct varatt_external is a traditional "TOAST pointer", that is, the * information needed to fetch a Datum stored out-of-line in a TOAST table. - * The data is compressed if and only if va_extsize < va_rawsize - VARHDRSZ. + * The data is compressed if and only if size in va_extinfo < va_rawsize - VARHDRSZ. * This struct must not contain any padding, because we sometimes compare * these pointers using memcmp. * @@ -67,7 +67,8 @@ typedef struct varatt_external { int32 va_rawsize; /* Original data size (includes header) */ - int32 va_extsize; /* External saved size (doesn't) */ + uint32 va_extinfo; /* External saved size (without header) and + * flags */ Oid va_valueid; /* Unique ID of value within TOAST table */ Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external; @@ -145,9 +146,18 @@ typedef union struct /* Compressed-in-line format */ { uint32 va_header; - uint32 va_rawsize; /* Original data size (excludes header) */ + uint32 va_info; /* Original data size (excludes header) and + * flags */ char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ } va_compressed; + struct /* Compressed-in-line format */ + { + uint32 va_header; + uint32 va_info; /* Original data size (excludes header) and + * flags */ + Oid va_cmid; /* Oid of compression options */ + char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ + } va_custom_compressed; } varattrib_4b; typedef struct @@ -280,8 +290,14 @@ typedef struct #define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) #define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data) +/* va_info in va_compress contains raw size of datum and optional flags */ #define VARRAWSIZE_4B_C(PTR) \ - (((varattrib_4b *) (PTR))->va_compressed.va_rawsize) + (((varattrib_4b *) (PTR))->va_compressed.va_info & 0x3FFFFFFF) +#define VARFLAGS_4B_C(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_info >> 30) + +#define VARHDRSZ_CUSTOM_COMPRESSED \ + (offsetof(varattrib_4b, va_custom_compressed.va_data)) /* Externally visible macros */ @@ -310,6 +326,8 @@ typedef struct #define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR) #define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) +#define VARATT_IS_CUSTOM_COMPRESSED(PTR) (VARATT_IS_4B_C(PTR) && \ + (VARFLAGS_4B_C(PTR) == 0x02)) #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) #define VARATT_IS_EXTERNAL_ONDISK(PTR) \ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 4f333586ee..48051b8294 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -37,6 +37,7 @@ enum SysCacheIdentifier AMOPOPID, AMOPSTRATEGY, AMPROCNUM, + ATTCOMPRESSIONOID, ATTNAME, ATTNUM, AUTHMEMMEMROLE, diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index e606a5fda4..37b9ddc3d7 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -438,10 +438,10 @@ begin end $$ language plpgsql immutable; alter table check_con_tbl add check (check_con_function(check_con_tbl.*)); \d+ check_con_tbl - Table "public.check_con_tbl" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | + Table "public.check_con_tbl" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | Check constraints: "check_con_tbl_check" CHECK (check_con_function(check_con_tbl.*)) diff --git a/src/test/regress/expected/create_cm.out b/src/test/regress/expected/create_cm.out new file mode 100644 index 0000000000..af23cdb5fd --- /dev/null +++ b/src/test/regress/expected/create_cm.out @@ -0,0 +1,379 @@ +-- test drop +DROP ACCESS METHOD pglz; --fail +ERROR: cannot drop access method pglz because it is required by the database system +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE droptest(d1 TEXT COMPRESSION pglz1); +DROP ACCESS METHOD pglz1; +ERROR: cannot drop access method pglz1 because other objects depend on it +DETAIL: table droptest column d1 depends on access method pglz1 +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP ACCESS METHOD pglz1 CASCADE; +NOTICE: drop cascades to table droptest column d1 +\d+ droptest + Table "public.droptest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+---------+-------------+--------------+------------- + +DROP TABLE droptest; +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +-- test auto drop of related attribute compression +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE cmaltertest (f1 TEXT COMPRESSION pglz2); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; + acname | acattnum | acoptions +--------+----------+----------- + pglz2 | 1 | +(1 row) + +ALTER TABLE cmaltertest DROP COLUMN f1; +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +DROP TABLE cmaltertest; +-- test drop +DROP ACCESS METHOD pglz2; +-- test alter data type +CREATE TABLE cmaltertest(f1 TEXT); +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +ALTER TABLE cmaltertest ALTER COLUMN f1 SET DATA TYPE INT USING f1::INTEGER; +\d+ cmaltertest + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +ALTER TABLE cmaltertest ALTER COLUMN f1 SET DATA TYPE TEXT; +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz +(1 row) + +DROP TABLE cmaltertest; +-- test storages +CREATE TABLE cmstoragetest(f1 TEXT, f2 INT); +ALTER TABLE cmstoragetest ALTER COLUMN f2 + SET COMPRESSION pglz WITH (min_input_size '100'); -- fail +ERROR: column data type integer does not support compression +ALTER TABLE cmstoragetest ALTER COLUMN f1 + SET COMPRESSION pglz WITH (min_input_size '100', min_comp_rate '50'); +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE EXTERNAL; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + f1 | text | | | | external | pglz(min_comp_rate '50', min_input_size '100') | | + f2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE MAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + f1 | text | | | | main | pglz(min_comp_rate '50', min_input_size '100') | | + f2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE PLAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + f1 | text | | | | plain | pglz(min_comp_rate '50', min_input_size '100') | | + f2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE EXTENDED; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + f1 | text | | | | extended | pglz(min_comp_rate '50', min_input_size '100') | | + f2 | integer | | | | plain | | | + +DROP TABLE cmstoragetest; +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +-- test PRESERVE +CREATE TABLE cmtest(f1 TEXT); +\d+ cmtest + Table "public.cmtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +-- view to check dependencies +CREATE VIEW cmtest_deps AS + SELECT classid, objsubid, refclassid, refobjsubid, deptype + FROM pg_depend + WHERE (refclassid = 4001 OR classid = 4001) AND + (objid = 'cmtest'::regclass OR refobjid = 'cmtest'::regclass) + ORDER by objid, refobjid; +INSERT INTO cmtest VALUES(repeat('1234567890',1001)); +-- one normal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 4001 | 0 | n +(1 row) + +-- check decompression +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +CREATE FUNCTION on_cmtest_rewrite() +RETURNS event_trigger AS $$ +BEGIN + RAISE NOTICE 'cmtest rewrite'; +END; +$$ LANGUAGE plpgsql; +CREATE EVENT TRIGGER notice_on_cmtest_rewrite ON table_rewrite + EXECUTE PROCEDURE on_cmtest_rewrite(); +-- no rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1002)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 +(2 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- one normal and one internal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 4001 | 0 | n + 4001 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz2 PRESERVE (pglz1); +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1003)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 +(3 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz2, pglz1 +(1 row) + +-- two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 4001 | 0 | 1259 | 1 | i + 4001 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz; +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1004)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 + 10040 +(4 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz +(1 row) + +-- one nornal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 4001 | 0 | n +(1 row) + +-- no rewrites +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1005)); +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz + WITH (min_input_size '1000') PRESERVE (pglz, pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1006)); +-- one nornal dependency and two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 4001 | 0 | n + 4001 | 0 | 1259 | 1 | i + 4001 | 0 | 1259 | 1 | i +(3 rows) + +-- remove function and related event trigger +DROP FUNCTION on_cmtest_rewrite CASCADE; +NOTICE: drop cascades to event trigger notice_on_cmtest_rewrite +-- test moving +CREATE TABLE cmdata(f1 text COMPRESSION pglz WITH (first_success_by '10000')); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +INSERT INTO cmdata VALUES(repeat('1234567890',1001)); +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +-- we update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove3 SELECT * FROM cmdata; +-- drop original compression information +DROP TABLE cmdata; +-- check data is ok +SELECT length(f1) FROM cmmove1; + length +-------- + 10000 + 10010 +(2 rows) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 + 10010 +(2 rows) + +-- create different types of tables +CREATE TABLE cmexample (f1 TEXT COMPRESSION pglz1); +CREATE TABLE cmtestlike1 (LIKE cmexample INCLUDING COMPRESSION); +CREATE TABLE cmtestlike2 (f2 INT) INHERITS (cmexample); +\d+ cmtestlike1 + Table "public.cmtestlike1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + +\d+ cmtestlike2 + Table "public.cmtestlike2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + f2 | integer | | | | plain | | | +Inherits: cmexample + +-- test two columns +CREATE TABLE cmaltertest(f1 TEXT, f2 TEXT COMPRESSION pglz1); +\d+ cmaltertest; + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + f2 | text | | | | extended | pglz1 | | + +-- fail, changing one column twice +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz, + ALTER COLUMN f1 SET COMPRESSION pglz; +ERROR: cannot alter compression of column "f1" twice +HINT: Remove one of statements from the command. +-- with rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1, + ALTER COLUMN f2 SET COMPRESSION pglz PRESERVE (pglz1); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- no rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (pglz1), + ALTER COLUMN f2 SET COMPRESSION pglz2 PRESERVE (pglz1, pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz2, pglz1 +(1 row) + +-- make pglz2 droppable +ALTER TABLE cmaltertest ALTER COLUMN f2 SET COMPRESSION pglz1; +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass OR acrelid = 'cmtest'::regclass; + acname | acattnum | acoptions +--------+----------+----------------------- + pglz1 | 1 | + pglz | 1 | {min_input_size=1000} + pglz1 | 2 | + pglz1 | 1 | +(4 rows) + +DROP ACCESS METHOD pglz2; +DROP VIEW cmtest_deps; +DROP TABLE cmmove1, cmmove2, cmmove3; +DROP TABLE cmexample, cmtestlike1, cmtestlike2; diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 39a963888d..ed4e844d6c 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -425,11 +425,11 @@ Partition key: RANGE (a oid_ops, plusone(b), c, d COLLATE "C") Number of partitions: 0 \d+ partitioned2 - Table "public.partitioned2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | integer | | | | plain | | - b | text | | | | extended | | + Table "public.partitioned2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | text | | | | extended | pglz | | Partition key: RANGE (((a + 1)), substr(b, 1, 5)) Number of partitions: 0 @@ -438,11 +438,11 @@ ERROR: no partition of relation "partitioned2" found for row DETAIL: Partition key of the failing row contains ((a + 1), substr(b, 1, 5)) = (2, hello). CREATE TABLE part2_1 PARTITION OF partitioned2 FOR VALUES FROM (-1, 'aaaaa') TO (100, 'ccccc'); \d+ part2_1 - Table "public.part2_1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | integer | | | | plain | | - b | text | | | | extended | | + Table "public.part2_1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | text | | | | extended | pglz | | Partition of: partitioned2 FOR VALUES FROM ('-1', 'aaaaa') TO (100, 'ccccc') Partition constraint: (((a + 1) IS NOT NULL) AND (substr(b, 1, 5) IS NOT NULL) AND (((a + 1) > '-1'::integer) OR (((a + 1) = '-1'::integer) AND (substr(b, 1, 5) >= 'aaaaa'::text))) AND (((a + 1) < 100) OR (((a + 1) = 100) AND (substr(b, 1, 5) < 'ccccc'::text)))) @@ -600,10 +600,10 @@ CREATE TABLE oids_parted ( ) PARTITION BY RANGE (a) WITH OIDS; CREATE TABLE part_forced_oids PARTITION OF oids_parted FOR VALUES FROM (1) TO (10) WITHOUT OIDS; \d+ part_forced_oids - Table "public.part_forced_oids" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | + Table "public.part_forced_oids" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | Partition of: oids_parted FOR VALUES FROM (1) TO (10) Partition constraint: ((a IS NOT NULL) AND (a >= 1) AND (a < 10)) Has OIDs: yes @@ -739,11 +739,11 @@ CREATE TABLE part_c PARTITION OF parted (b WITH OPTIONS NOT NULL DEFAULT 0) FOR CREATE TABLE part_c_1_10 PARTITION OF part_c FOR VALUES FROM (1) TO (10); -- Partition bound in describe output \d+ part_b - Table "public.part_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 1 | plain | | + Table "public.part_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 1 | plain | | | Partition of: parted FOR VALUES IN ('b') Partition constraint: ((a IS NOT NULL) AND (a = 'b'::text)) Check constraints: @@ -752,11 +752,11 @@ Check constraints: -- Both partition bound and partition key in describe output \d+ part_c - Table "public.part_c" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 0 | plain | | + Table "public.part_c" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 0 | plain | | | Partition of: parted FOR VALUES IN ('c') Partition constraint: ((a IS NOT NULL) AND (a = 'c'::text)) Partition key: RANGE (b) @@ -766,11 +766,11 @@ Partitions: part_c_1_10 FOR VALUES FROM (1) TO (10) -- a level-2 partition's constraint will include the parent's expressions \d+ part_c_1_10 - Table "public.part_c_1_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 0 | plain | | + Table "public.part_c_1_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 0 | plain | | | Partition of: part_c FOR VALUES FROM (1) TO (10) Partition constraint: ((a IS NOT NULL) AND (a = 'c'::text) AND (b IS NOT NULL) AND (b >= 1) AND (b < 10)) Check constraints: @@ -803,46 +803,46 @@ Number of partitions: 3 (Use \d+ to list them.) CREATE TABLE range_parted4 (a int, b int, c int) PARTITION BY RANGE (abs(a), abs(b), c); CREATE TABLE unbounded_range_part PARTITION OF range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (MAXVALUE, MAXVALUE, MAXVALUE); \d+ unbounded_range_part - Table "public.unbounded_range_part" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.unbounded_range_part" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (MAXVALUE, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL)) DROP TABLE unbounded_range_part; CREATE TABLE range_parted4_1 PARTITION OF range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (1, MAXVALUE, MAXVALUE); \d+ range_parted4_1 - Table "public.range_parted4_1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (1, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND (abs(a) <= 1)) CREATE TABLE range_parted4_2 PARTITION OF range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, MAXVALUE); \d+ range_parted4_2 - Table "public.range_parted4_2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 3) OR ((abs(a) = 3) AND (abs(b) > 4)) OR ((abs(a) = 3) AND (abs(b) = 4) AND (c >= 5))) AND ((abs(a) < 6) OR ((abs(a) = 6) AND (abs(b) <= 7)))) CREATE TABLE range_parted4_3 PARTITION OF range_parted4 FOR VALUES FROM (6, 8, MINVALUE) TO (9, MAXVALUE, MAXVALUE); \d+ range_parted4_3 - Table "public.range_parted4_3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (6, 8, MINVALUE) TO (9, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 6) OR ((abs(a) = 6) AND (abs(b) >= 8))) AND (abs(a) <= 9)) @@ -862,11 +862,11 @@ SELECT obj_description('parted_col_comment'::regclass); (1 row) \d+ parted_col_comment - Table "public.parted_col_comment" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+--------------- - a | integer | | | | plain | | Partition key - b | text | | | | extended | | + Table "public.parted_col_comment" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+--------------- + a | integer | | | | plain | | | Partition key + b | text | | | | extended | pglz | | Partition key: LIST (a) Number of partitions: 0 @@ -875,10 +875,10 @@ DROP TABLE parted_col_comment; CREATE TABLE arrlp (a int[]) PARTITION BY LIST (a); CREATE TABLE arrlp12 PARTITION OF arrlp FOR VALUES IN ('{1}', '{2}'); \d+ arrlp12 - Table "public.arrlp12" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+----------+--------------+------------- - a | integer[] | | | | extended | | + Table "public.arrlp12" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-----------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer[] | | | | extended | pglz | | Partition of: arrlp FOR VALUES IN ('{1}', '{2}') Partition constraint: ((a IS NOT NULL) AND (((a)::anyarray OPERATOR(pg_catalog.=) '{1}'::integer[]) OR ((a)::anyarray OPERATOR(pg_catalog.=) '{2}'::integer[]))) diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index 8d4543bfe8..f046eee45f 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -158,32 +158,32 @@ CREATE TABLE ctlt4 (a text, c text); ALTER TABLE ctlt4 ALTER COLUMN c SET STORAGE EXTERNAL; CREATE TABLE ctlt12_storage (LIKE ctlt1 INCLUDING STORAGE, LIKE ctlt2 INCLUDING STORAGE); \d+ ctlt12_storage - Table "public.ctlt12_storage" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | - b | text | | | | extended | | - c | text | | | | external | | + Table "public.ctlt12_storage" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | CREATE TABLE ctlt12_comments (LIKE ctlt1 INCLUDING COMMENTS, LIKE ctlt2 INCLUDING COMMENTS); \d+ ctlt12_comments - Table "public.ctlt12_comments" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | extended | | A - b | text | | | | extended | | B - c | text | | | | extended | | C + Table "public.ctlt12_comments" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | extended | pglz | | A + b | text | | | | extended | pglz | | B + c | text | | | | extended | pglz | | C CREATE TABLE ctlt1_inh (LIKE ctlt1 INCLUDING CONSTRAINTS INCLUDING COMMENTS) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition NOTICE: merging column "b" with inherited definition NOTICE: merging constraint "ctlt1_a_check" with inherited definition \d+ ctlt1_inh - Table "public.ctlt1_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A - b | text | | | | extended | | B + Table "public.ctlt1_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A + b | text | | | | extended | pglz | | B Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) Inherits: ctlt1 @@ -197,12 +197,12 @@ SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_con CREATE TABLE ctlt13_inh () INHERITS (ctlt1, ctlt3); NOTICE: merging multiple inherited definitions of column "a" \d+ ctlt13_inh - Table "public.ctlt13_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | - b | text | | | | extended | | - c | text | | | | external | | + Table "public.ctlt13_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) @@ -212,12 +212,12 @@ Inherits: ctlt1, CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition \d+ ctlt13_like - Table "public.ctlt13_like" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A3 - b | text | | | | extended | | - c | text | | | | external | | C + Table "public.ctlt13_like" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A3 + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | C Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) @@ -231,11 +231,11 @@ SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_con CREATE TABLE ctlt_all (LIKE ctlt1 INCLUDING ALL); \d+ ctlt_all - Table "public.ctlt_all" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A - b | text | | | | extended | | B + Table "public.ctlt_all" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A + b | text | | | | extended | pglz | | B Indexes: "ctlt_all_pkey" PRIMARY KEY, btree (a) "ctlt_all_b_idx" btree (b) diff --git a/src/test/regress/expected/domain.out b/src/test/regress/expected/domain.out index f4eebb75cf..11924864e9 100644 --- a/src/test/regress/expected/domain.out +++ b/src/test/regress/expected/domain.out @@ -272,10 +272,10 @@ explain (verbose, costs off) create rule silly as on delete to dcomptable do instead update dcomptable set d1.r = (d1).r - 1, d1.i = (d1).i + 1 where (d1).i > 0; \d+ dcomptable - Table "public.dcomptable" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+----------+--------------+------------- - d1 | dcomptype | | | | extended | | + Table "public.dcomptable" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-----------+-----------+----------+---------+----------+-------------+--------------+------------- + d1 | dcomptype | | | | extended | pglz | | Indexes: "dcomptable_d1_key" UNIQUE CONSTRAINT, btree (d1) Rules: @@ -409,10 +409,10 @@ create rule silly as on delete to dcomptable do instead update dcomptable set d1[1].r = d1[1].r - 1, d1[1].i = d1[1].i + 1 where d1[1].i > 0; \d+ dcomptable - Table "public.dcomptable" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------------+-----------+----------+---------+----------+--------------+------------- - d1 | dcomptypea | | | | extended | | + Table "public.dcomptable" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------------+-----------+----------+---------+----------+-------------+--------------+------------- + d1 | dcomptypea | | | | extended | pglz | | Indexes: "dcomptable_d1_key" UNIQUE CONSTRAINT, btree (d1) Rules: diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index 7b54e96d30..9faf540236 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -1330,12 +1330,12 @@ CREATE TABLE fd_pt1 ( CREATE FOREIGN TABLE ft2 () INHERITS (fd_pt1) SERVER s0 OPTIONS (delimiter ',', quote '"', "be quoted" 'value'); \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1351,12 +1351,12 @@ Inherits: fd_pt1 DROP FOREIGN TABLE ft2; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | CREATE FOREIGN TABLE ft2 ( c1 integer NOT NULL, @@ -1375,12 +1375,12 @@ FDW options: (delimiter ',', quote '"', "be quoted" 'value') ALTER FOREIGN TABLE ft2 INHERIT fd_pt1; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1418,12 +1418,12 @@ Child tables: ct3, ft3 \d+ ct3 - Table "public.ct3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.ct3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Inherits: ft2 \d+ ft3 @@ -1443,17 +1443,17 @@ ALTER TABLE fd_pt1 ADD COLUMN c6 integer; ALTER TABLE fd_pt1 ADD COLUMN c7 integer NOT NULL; ALTER TABLE fd_pt1 ADD COLUMN c8 integer; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | | plain | | - c5 | integer | | | 0 | plain | | - c6 | integer | | | | plain | | - c7 | integer | | not null | | plain | | - c8 | integer | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | | plain | | | + c5 | integer | | | 0 | plain | | | + c6 | integer | | | | plain | | | + c7 | integer | | not null | | plain | | | + c8 | integer | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1475,17 +1475,17 @@ Child tables: ct3, ft3 \d+ ct3 - Table "public.ct3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | | plain | | - c5 | integer | | | 0 | plain | | - c6 | integer | | | | plain | | - c7 | integer | | not null | | plain | | - c8 | integer | | | | plain | | + Table "public.ct3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | | plain | | | + c5 | integer | | | 0 | plain | | | + c6 | integer | | | | plain | | | + c7 | integer | | not null | | plain | | | + c8 | integer | | | | plain | | | Inherits: ft2 \d+ ft3 @@ -1517,17 +1517,17 @@ ALTER TABLE fd_pt1 ALTER COLUMN c1 SET (n_distinct = 100); ALTER TABLE fd_pt1 ALTER COLUMN c8 SET STATISTICS -1; ALTER TABLE fd_pt1 ALTER COLUMN c8 SET STORAGE EXTERNAL; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | 0 | plain | | - c5 | integer | | | | plain | | - c6 | integer | | not null | | plain | | - c7 | integer | | | | plain | | - c8 | text | | | | external | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | 0 | plain | | | + c5 | integer | | | | plain | | | + c6 | integer | | not null | | plain | | | + c7 | integer | | | | plain | | | + c8 | text | | | | external | pglz | | Child tables: ft2 \d+ ft2 @@ -1555,12 +1555,12 @@ ALTER TABLE fd_pt1 DROP COLUMN c6; ALTER TABLE fd_pt1 DROP COLUMN c7; ALTER TABLE fd_pt1 DROP COLUMN c8; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1592,12 +1592,12 @@ SELECT relname, conname, contype, conislocal, coninhcount, connoinherit -- child does not inherit NO INHERIT constraints \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk1" CHECK (c1 > 0) NO INHERIT "fd_pt1chk2" CHECK (c2 <> ''::text) @@ -1636,12 +1636,12 @@ ALTER FOREIGN TABLE ft2 ADD CONSTRAINT fd_pt1chk2 CHECK (c2 <> ''); ALTER FOREIGN TABLE ft2 INHERIT fd_pt1; -- child does not inherit NO INHERIT constraints \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk1" CHECK (c1 > 0) NO INHERIT "fd_pt1chk2" CHECK (c2 <> ''::text) @@ -1667,12 +1667,12 @@ ALTER TABLE fd_pt1 DROP CONSTRAINT fd_pt1chk2 CASCADE; INSERT INTO fd_pt1 VALUES (1, 'fd_pt1'::text, '1994-01-01'::date); ALTER TABLE fd_pt1 ADD CONSTRAINT fd_pt1chk3 CHECK (c2 <> '') NOT VALID; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) NOT VALID Child tables: ft2 @@ -1694,12 +1694,12 @@ Inherits: fd_pt1 -- VALIDATE CONSTRAINT need do nothing on foreign tables ALTER TABLE fd_pt1 VALIDATE CONSTRAINT fd_pt1chk3; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) Child tables: ft2 @@ -1721,12 +1721,12 @@ Inherits: fd_pt1 -- OID system column ALTER TABLE fd_pt1 SET WITH OIDS; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) Child tables: ft2 @@ -1751,12 +1751,12 @@ ALTER TABLE ft2 SET WITHOUT OIDS; -- ERROR ERROR: cannot drop inherited column "oid" ALTER TABLE fd_pt1 SET WITHOUT OIDS; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) Child tables: ft2 @@ -1782,12 +1782,12 @@ ALTER TABLE fd_pt1 RENAME COLUMN c3 TO f3; -- changes name of a constraint recursively ALTER TABLE fd_pt1 RENAME CONSTRAINT fd_pt1chk3 TO f2_check; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | not null | | plain | 10000 | - f2 | text | | | | extended | | - f3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | not null | | plain | | 10000 | + f2 | text | | | | extended | pglz | | + f3 | date | | | | plain | | | Check constraints: "f2_check" CHECK (f2 <> ''::text) Child tables: ft2 @@ -1846,12 +1846,12 @@ CREATE TABLE fd_pt2 ( CREATE FOREIGN TABLE fd_pt2_1 PARTITION OF fd_pt2 FOR VALUES IN (1) SERVER s0 OPTIONS (delimiter ',', quote '"', "be quoted" 'value'); \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1891,12 +1891,12 @@ ERROR: table "fd_pt2_1" contains column "c4" not found in parent "fd_pt2" DETAIL: The new partition may contain only the columns present in parent. DROP FOREIGN TABLE fd_pt2_1; \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Number of partitions: 0 @@ -1918,12 +1918,12 @@ FDW options: (delimiter ',', quote '"', "be quoted" 'value') -- no attach partition validation occurs for foreign tables ALTER TABLE fd_pt2 ATTACH PARTITION fd_pt2_1 FOR VALUES IN (1); \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1946,12 +1946,12 @@ ERROR: cannot add column to a partition ALTER TABLE fd_pt2_1 ALTER c3 SET NOT NULL; ALTER TABLE fd_pt2_1 ADD CONSTRAINT p21chk CHECK (c2 <> ''); \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1976,12 +1976,12 @@ ERROR: column "c1" is marked NOT NULL in parent table ALTER TABLE fd_pt2 DETACH PARTITION fd_pt2_1; ALTER TABLE fd_pt2 ALTER c2 SET NOT NULL; \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | not null | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | not null | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Number of partitions: 0 @@ -2004,12 +2004,12 @@ ALTER TABLE fd_pt2 ATTACH PARTITION fd_pt2_1 FOR VALUES IN (1); ALTER TABLE fd_pt2 DETACH PARTITION fd_pt2_1; ALTER TABLE fd_pt2 ADD CONSTRAINT fd_pt2chk1 CHECK (c1 > 0); \d+ fd_pt2 - Table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | not null | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | not null | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Check constraints: "fd_pt2chk1" CHECK (c1 > 0) diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index f56151fc1e..d4215af768 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -1023,13 +1023,13 @@ ALTER TABLE inhts RENAME aa TO aaa; -- to be failed ERROR: cannot rename inherited column "aa" ALTER TABLE inhts RENAME d TO dd; \d+ inhts - Table "public.inhts" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aa | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | - dd | integer | | | | plain | | + Table "public.inhts" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aa | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | + dd | integer | | | | plain | | | Inherits: inht1, inhs1 @@ -1042,14 +1042,14 @@ NOTICE: merging multiple inherited definitions of column "aa" NOTICE: merging multiple inherited definitions of column "b" ALTER TABLE inht1 RENAME aa TO aaa; \d+ inht4 - Table "public.inht4" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aaa | integer | | | | plain | | - b | integer | | | | plain | | - x | integer | | | | plain | | - y | integer | | | | plain | | - z | integer | | | | plain | | + Table "public.inht4" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aaa | integer | | | | plain | | | + b | integer | | | | plain | | | + x | integer | | | | plain | | | + y | integer | | | | plain | | | + z | integer | | | | plain | | | Inherits: inht2, inht3 @@ -1059,14 +1059,14 @@ ALTER TABLE inht1 RENAME aaa TO aaaa; ALTER TABLE inht1 RENAME b TO bb; -- to be failed ERROR: cannot rename inherited column "b" \d+ inhts - Table "public.inhts" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aaaa | integer | | | | plain | | - b | integer | | | | plain | | - x | integer | | | | plain | | - c | integer | | | | plain | | - d | integer | | | | plain | | + Table "public.inhts" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aaaa | integer | | | | plain | | | + b | integer | | | | plain | | | + x | integer | | | | plain | | | + c | integer | | | | plain | | | + d | integer | | | | plain | | | Inherits: inht2, inhs1 @@ -1106,33 +1106,33 @@ drop cascades to table inht4 CREATE TABLE test_constraints (id int, val1 varchar, val2 int, UNIQUE(val1, val2)); CREATE TABLE test_constraints_inh () INHERITS (test_constraints); \d+ test_constraints - Table "public.test_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Indexes: "test_constraints_val1_val2_key" UNIQUE CONSTRAINT, btree (val1, val2) Child tables: test_constraints_inh ALTER TABLE ONLY test_constraints DROP CONSTRAINT test_constraints_val1_val2_key; \d+ test_constraints - Table "public.test_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Child tables: test_constraints_inh \d+ test_constraints_inh - Table "public.test_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Inherits: test_constraints DROP TABLE test_constraints_inh; @@ -1143,27 +1143,27 @@ CREATE TABLE test_ex_constraints ( ); CREATE TABLE test_ex_constraints_inh () INHERITS (test_ex_constraints); \d+ test_ex_constraints - Table "public.test_ex_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Indexes: "test_ex_constraints_c_excl" EXCLUDE USING gist (c WITH &&) Child tables: test_ex_constraints_inh ALTER TABLE test_ex_constraints DROP CONSTRAINT test_ex_constraints_c_excl; \d+ test_ex_constraints - Table "public.test_ex_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Child tables: test_ex_constraints_inh \d+ test_ex_constraints_inh - Table "public.test_ex_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Inherits: test_ex_constraints DROP TABLE test_ex_constraints_inh; @@ -1173,37 +1173,37 @@ CREATE TABLE test_primary_constraints(id int PRIMARY KEY); CREATE TABLE test_foreign_constraints(id1 int REFERENCES test_primary_constraints(id)); CREATE TABLE test_foreign_constraints_inh () INHERITS (test_foreign_constraints); \d+ test_primary_constraints - Table "public.test_primary_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id | integer | | not null | | plain | | + Table "public.test_primary_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id | integer | | not null | | plain | | | Indexes: "test_primary_constraints_pkey" PRIMARY KEY, btree (id) Referenced by: TABLE "test_foreign_constraints" CONSTRAINT "test_foreign_constraints_id1_fkey" FOREIGN KEY (id1) REFERENCES test_primary_constraints(id) \d+ test_foreign_constraints - Table "public.test_foreign_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Foreign-key constraints: "test_foreign_constraints_id1_fkey" FOREIGN KEY (id1) REFERENCES test_primary_constraints(id) Child tables: test_foreign_constraints_inh ALTER TABLE test_foreign_constraints DROP CONSTRAINT test_foreign_constraints_id1_fkey; \d+ test_foreign_constraints - Table "public.test_foreign_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Child tables: test_foreign_constraints_inh \d+ test_foreign_constraints_inh - Table "public.test_foreign_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Inherits: test_foreign_constraints DROP TABLE test_foreign_constraints_inh; diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out index 97419a744f..b33f8f93e2 100644 --- a/src/test/regress/expected/insert.out +++ b/src/test/regress/expected/insert.out @@ -142,11 +142,11 @@ create rule irule3 as on insert to inserttest2 do also insert into inserttest (f4[1].if1, f4[1].if2[2]) select new.f1, new.f2; \d+ inserttest2 - Table "public.inserttest2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+----------+--------------+------------- - f1 | bigint | | | | plain | | - f2 | text | | | | extended | | + Table "public.inserttest2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | bigint | | | | plain | | | + f2 | text | | | | extended | pglz | | Rules: irule1 AS ON INSERT TO inserttest2 DO INSERT INTO inserttest (f3.if2[1], f3.if2[2]) @@ -432,11 +432,11 @@ from hash_parted order by part; -- test \d+ output on a table which has both partitioned and unpartitioned -- partitions \d+ list_parted - Table "public.list_parted" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.list_parted" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition key: LIST (lower(a)) Partitions: part_aa_bb FOR VALUES IN ('aa', 'bb'), part_cc_dd FOR VALUES IN ('cc', 'dd'), @@ -456,10 +456,10 @@ drop function dummy_hashint4(a int4, seed int8); create table list_parted (a int) partition by list (a); create table part_default partition of list_parted default; \d+ part_default - Table "public.part_default" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | + Table "public.part_default" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | Partition of: list_parted DEFAULT No partition constraint @@ -785,11 +785,11 @@ create table mcrparted6_common_ge_10 partition of mcrparted for values from ('co create table mcrparted7_gt_common_lt_d partition of mcrparted for values from ('common', maxvalue) to ('d', minvalue); create table mcrparted8_ge_d partition of mcrparted for values from ('d', minvalue) to (maxvalue, maxvalue); \d+ mcrparted - Table "public.mcrparted" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition key: RANGE (a, b) Partitions: mcrparted1_lt_b FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVALUE), mcrparted2_b FOR VALUES FROM ('b', MINVALUE) TO ('c', MINVALUE), @@ -801,74 +801,74 @@ Partitions: mcrparted1_lt_b FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVAL mcrparted8_ge_d FOR VALUES FROM ('d', MINVALUE) TO (MAXVALUE, MAXVALUE) \d+ mcrparted1_lt_b - Table "public.mcrparted1_lt_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted1_lt_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a < 'b'::text)) \d+ mcrparted2_b - Table "public.mcrparted2_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted2_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('b', MINVALUE) TO ('c', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'b'::text) AND (a < 'c'::text)) \d+ mcrparted3_c_to_common - Table "public.mcrparted3_c_to_common" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted3_c_to_common" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('c', MINVALUE) TO ('common', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'c'::text) AND (a < 'common'::text)) \d+ mcrparted4_common_lt_0 - Table "public.mcrparted4_common_lt_0" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted4_common_lt_0" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', MINVALUE) TO ('common', 0) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b < 0)) \d+ mcrparted5_common_0_to_10 - Table "public.mcrparted5_common_0_to_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted5_common_0_to_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', 0) TO ('common', 10) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b >= 0) AND (b < 10)) \d+ mcrparted6_common_ge_10 - Table "public.mcrparted6_common_ge_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted6_common_ge_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', 10) TO ('common', MAXVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b >= 10)) \d+ mcrparted7_gt_common_lt_d - Table "public.mcrparted7_gt_common_lt_d" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted7_gt_common_lt_d" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', MAXVALUE) TO ('d', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a > 'common'::text) AND (a < 'd'::text)) \d+ mcrparted8_ge_d - Table "public.mcrparted8_ge_d" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted8_ge_d" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('d', MINVALUE) TO (MAXVALUE, MAXVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'd'::text)) diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 01608d2c04..bd85b93d3e 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1724,7 +1724,9 @@ WHERE p1.amhandler = 0; SELECT p1.oid, p1.amname, p2.oid, p2.proname FROM pg_am AS p1, pg_proc AS p2 WHERE p2.oid = p1.amhandler AND - (p2.prorettype != 'index_am_handler'::regtype OR p2.proretset + ((p2.prorettype != 'index_am_handler'::regtype + AND p2.prorettype != 'compression_am_handler'::regtype) + OR p2.proretset OR p2.pronargs != 1 OR p2.proargtypes[0] != 'internal'::regtype); oid | amname | oid | proname diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 0c86c647bc..a7744bb6f2 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -65,11 +65,11 @@ SELECT pubname, puballtables FROM pg_publication WHERE pubname = 'testpub_forall (1 row) \d+ testpub_tbl2 - Table "public.testpub_tbl2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl2_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl2_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl2_pkey" PRIMARY KEY, btree (id) Publications: @@ -141,22 +141,22 @@ ALTER PUBLICATION testpub_default SET TABLE testpub_tbl1; ALTER PUBLICATION testpub_default ADD TABLE pub_test.testpub_nopk; ALTER PUBLICATION testpib_ins_trunct ADD TABLE pub_test.testpub_nopk, testpub_tbl1; \d+ pub_test.testpub_nopk - Table "pub_test.testpub_nopk" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - foo | integer | | | | plain | | - bar | integer | | | | plain | | + Table "pub_test.testpub_nopk" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + foo | integer | | | | plain | | | + bar | integer | | | | plain | | | Publications: "testpib_ins_trunct" "testpub_default" "testpub_fortbl" \d+ testpub_tbl1 - Table "public.testpub_tbl1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl1_pkey" PRIMARY KEY, btree (id) Publications: @@ -178,11 +178,11 @@ ALTER PUBLICATION testpub_default DROP TABLE testpub_tbl1, pub_test.testpub_nopk ALTER PUBLICATION testpub_default DROP TABLE pub_test.testpub_nopk; ERROR: relation "testpub_nopk" is not part of the publication \d+ testpub_tbl1 - Table "public.testpub_tbl1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl1_pkey" PRIMARY KEY, btree (id) Publications: diff --git a/src/test/regress/expected/replica_identity.out b/src/test/regress/expected/replica_identity.out index 67c34a92a4..b86883ce2d 100644 --- a/src/test/regress/expected/replica_identity.out +++ b/src/test/regress/expected/replica_identity.out @@ -158,13 +158,13 @@ SELECT relreplident FROM pg_class WHERE oid = 'test_replica_identity'::regclass; (1 row) \d+ test_replica_identity - Table "public.test_replica_identity" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('test_replica_identity_id_seq'::regclass) | plain | | - keya | text | | not null | | extended | | - keyb | text | | not null | | extended | | - nonkey | text | | | | extended | | + Table "public.test_replica_identity" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('test_replica_identity_id_seq'::regclass) | plain | | | + keya | text | | not null | | extended | pglz | | + keyb | text | | not null | | extended | pglz | | + nonkey | text | | | | extended | pglz | | Indexes: "test_replica_identity_pkey" PRIMARY KEY, btree (id) "test_replica_identity_expr" UNIQUE, btree (keya, keyb, (3)) diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out index f1ae40df61..fd4110b5f7 100644 --- a/src/test/regress/expected/rowsecurity.out +++ b/src/test/regress/expected/rowsecurity.out @@ -938,14 +938,14 @@ CREATE POLICY pp1 ON part_document AS PERMISSIVE CREATE POLICY pp1r ON part_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ part_document - Table "regress_rls_schema.part_document" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ----------+---------+-----------+----------+---------+----------+--------------+------------- - did | integer | | | | plain | | - cid | integer | | | | plain | | - dlevel | integer | | not null | | plain | | - dauthor | name | | | | plain | | - dtitle | text | | | | extended | | + Table "regress_rls_schema.part_document" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +---------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + did | integer | | | | plain | | | + cid | integer | | | | plain | | | + dlevel | integer | | not null | | plain | | | + dauthor | name | | | | plain | | | + dtitle | text | | | | extended | pglz | | Partition key: RANGE (cid) Policies: POLICY "pp1" diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 5149b72fe9..8b0baeb0c3 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2801,11 +2801,11 @@ select * from rules_log; create rule r3 as on delete to rules_src do notify rules_src_deletion; \d+ rules_src - Table "public.rules_src" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | - f2 | integer | | | | plain | | + Table "public.rules_src" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | integer | | | | plain | | | Rules: r1 AS ON UPDATE TO rules_src DO INSERT INTO rules_log (f1, f2, tag) VALUES (old.f1,old.f2,'old'::text), (new.f1,new.f2,'new'::text) @@ -2821,11 +2821,11 @@ Rules: create rule r4 as on insert to rules_src do instead insert into rules_log AS trgt SELECT NEW.* RETURNING trgt.f1, trgt.f2; create rule r5 as on update to rules_src do instead UPDATE rules_log AS trgt SET tag = 'updated' WHERE trgt.f1 = new.f1; \d+ rules_src - Table "public.rules_src" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | - f2 | integer | | | | plain | | + Table "public.rules_src" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | integer | | | | plain | | | Rules: r1 AS ON UPDATE TO rules_src DO INSERT INTO rules_log (f1, f2, tag) VALUES (old.f1,old.f2,'old'::text), (new.f1,new.f2,'new'::text) diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index ac0fb539e9..b23386ddb8 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -32,6 +32,8 @@ check2_tbl|f check_tbl|f circle_tbl|t city|f +cmaltertest|f +cmtest|f copy_tbl|f d|f d_star|f @@ -104,6 +106,7 @@ pg_aggregate|t pg_am|t pg_amop|t pg_amproc|t +pg_attr_compression|t pg_attrdef|t pg_attribute|t pg_auth_members|t diff --git a/src/test/regress/expected/update.out b/src/test/regress/expected/update.out index d09326c182..cdc93b2816 100644 --- a/src/test/regress/expected/update.out +++ b/src/test/regress/expected/update.out @@ -659,14 +659,14 @@ DROP TRIGGER d15_insert_trig ON part_d_15_20; :init_range_parted; create table part_def partition of range_parted default; \d+ part_def - Table "public.part_def" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | bigint | | | | plain | | - c | numeric | | | | main | | - d | integer | | | | plain | | - e | character varying | | | | extended | | + Table "public.part_def" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | bigint | | | | plain | | | + c | numeric | | | | main | pglz | | + d | integer | | | | plain | | | + e | character varying | | | | extended | pglz | | Partition of: range_parted DEFAULT Partition constraint: (NOT ((a IS NOT NULL) AND (b IS NOT NULL) AND (((a = 'a'::text) AND (b >= '1'::bigint) AND (b < '10'::bigint)) OR ((a = 'a'::text) AND (b >= '10'::bigint) AND (b < '20'::bigint)) OR ((a = 'b'::text) AND (b >= '1'::bigint) AND (b < '10'::bigint)) OR ((a = 'b'::text) AND (b >= '10'::bigint) AND (b < '20'::bigint)) OR ((a = 'b'::text) AND (b >= '20'::bigint) AND (b < '30'::bigint))))) diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index d308a05117..230ad1faa9 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -30,7 +30,7 @@ test: point lseg line box path polygon circle date time timetz timestamp timesta # geometry depends on point, lseg, box, path, polygon and circle # horology depends on interval, timetz, timestamp, timestamptz, reltime and abstime # ---------- -test: geometry horology regex oidjoins type_sanity opr_sanity misc_sanity comments expressions +test: geometry horology regex oidjoins type_sanity opr_sanity misc_sanity comments expressions create_cm # ---------- # These four each depend on the previous one diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 45147e9328..91a53699f0 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -43,6 +43,7 @@ test: inet test: macaddr test: macaddr8 test: tstypes +test: create_cm test: geometry test: horology test: regex diff --git a/src/test/regress/sql/create_cm.sql b/src/test/regress/sql/create_cm.sql new file mode 100644 index 0000000000..0c359c4aa9 --- /dev/null +++ b/src/test/regress/sql/create_cm.sql @@ -0,0 +1,185 @@ +-- test drop +DROP ACCESS METHOD pglz; --fail + +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE droptest(d1 TEXT COMPRESSION pglz1); +DROP ACCESS METHOD pglz1; +DROP ACCESS METHOD pglz1 CASCADE; +\d+ droptest +DROP TABLE droptest; + +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; + +-- test auto drop of related attribute compression +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE cmaltertest (f1 TEXT COMPRESSION pglz2); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; +ALTER TABLE cmaltertest DROP COLUMN f1; +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; +DROP TABLE cmaltertest; + +-- test drop +DROP ACCESS METHOD pglz2; + +-- test alter data type +CREATE TABLE cmaltertest(f1 TEXT); +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); +ALTER TABLE cmaltertest ALTER COLUMN f1 SET DATA TYPE INT USING f1::INTEGER; +\d+ cmaltertest +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass; +ALTER TABLE cmaltertest ALTER COLUMN f1 SET DATA TYPE TEXT; +SELECT pg_column_compression('cmaltertest', 'f1'); +DROP TABLE cmaltertest; + +-- test storages +CREATE TABLE cmstoragetest(f1 TEXT, f2 INT); +ALTER TABLE cmstoragetest ALTER COLUMN f2 + SET COMPRESSION pglz WITH (min_input_size '100'); -- fail +ALTER TABLE cmstoragetest ALTER COLUMN f1 + SET COMPRESSION pglz WITH (min_input_size '100', min_comp_rate '50'); +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE EXTERNAL; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE MAIN; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE PLAIN; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN f1 SET STORAGE EXTENDED; +\d+ cmstoragetest +DROP TABLE cmstoragetest; + +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; + +-- test PRESERVE +CREATE TABLE cmtest(f1 TEXT); +\d+ cmtest +-- view to check dependencies +CREATE VIEW cmtest_deps AS + SELECT classid, objsubid, refclassid, refobjsubid, deptype + FROM pg_depend + WHERE (refclassid = 4001 OR classid = 4001) AND + (objid = 'cmtest'::regclass OR refobjid = 'cmtest'::regclass) + ORDER by objid, refobjid; +INSERT INTO cmtest VALUES(repeat('1234567890',1001)); + +-- one normal dependency +SELECT * FROM cmtest_deps; + +-- check decompression +SELECT length(f1) FROM cmtest; +SELECT length(f1) FROM cmtest; + +CREATE FUNCTION on_cmtest_rewrite() +RETURNS event_trigger AS $$ +BEGIN + RAISE NOTICE 'cmtest rewrite'; +END; +$$ LANGUAGE plpgsql; + +CREATE EVENT TRIGGER notice_on_cmtest_rewrite ON table_rewrite + EXECUTE PROCEDURE on_cmtest_rewrite(); + +-- no rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1002)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- one normal and one internal dependency +SELECT * FROM cmtest_deps; + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz2 PRESERVE (pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1003)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- two internal dependencies +SELECT * FROM cmtest_deps; + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz; +INSERT INTO cmtest VALUES(repeat('1234567890',1004)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- one nornal dependency +SELECT * FROM cmtest_deps; + +-- no rewrites +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1005)); +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz + WITH (min_input_size '1000') PRESERVE (pglz, pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1006)); +-- one nornal dependency and two internal dependencies +SELECT * FROM cmtest_deps; + +-- remove function and related event trigger +DROP FUNCTION on_cmtest_rewrite CASCADE; + +-- test moving +CREATE TABLE cmdata(f1 text COMPRESSION pglz WITH (first_success_by '10000')); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +INSERT INTO cmdata VALUES(repeat('1234567890',1001)); + +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; + +-- we update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; + +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove3 SELECT * FROM cmdata; + +-- drop original compression information +DROP TABLE cmdata; + +-- check data is ok +SELECT length(f1) FROM cmmove1; +SELECT length(f1) FROM cmmove2; +SELECT length(f1) FROM cmmove3; + +-- create different types of tables +CREATE TABLE cmexample (f1 TEXT COMPRESSION pglz1); +CREATE TABLE cmtestlike1 (LIKE cmexample INCLUDING COMPRESSION); +CREATE TABLE cmtestlike2 (f2 INT) INHERITS (cmexample); + +\d+ cmtestlike1 +\d+ cmtestlike2 + +-- test two columns +CREATE TABLE cmaltertest(f1 TEXT, f2 TEXT COMPRESSION pglz1); +\d+ cmaltertest; +-- fail, changing one column twice +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz, + ALTER COLUMN f1 SET COMPRESSION pglz; + +-- with rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1, + ALTER COLUMN f2 SET COMPRESSION pglz PRESERVE (pglz1); +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +-- no rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (pglz1), + ALTER COLUMN f2 SET COMPRESSION pglz2 PRESERVE (pglz1, pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +-- make pglz2 droppable +ALTER TABLE cmaltertest ALTER COLUMN f2 SET COMPRESSION pglz1; +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::regclass OR acrelid = 'cmtest'::regclass; + +DROP ACCESS METHOD pglz2; +DROP VIEW cmtest_deps; +DROP TABLE cmmove1, cmmove2, cmmove3; +DROP TABLE cmexample, cmtestlike1, cmtestlike2; diff --git a/src/test/regress/sql/opr_sanity.sql b/src/test/regress/sql/opr_sanity.sql index a593d37643..9c2d8fe89d 100644 --- a/src/test/regress/sql/opr_sanity.sql +++ b/src/test/regress/sql/opr_sanity.sql @@ -1161,7 +1161,9 @@ WHERE p1.amhandler = 0; SELECT p1.oid, p1.amname, p2.oid, p2.proname FROM pg_am AS p1, pg_proc AS p2 WHERE p2.oid = p1.amhandler AND - (p2.prorettype != 'index_am_handler'::regtype OR p2.proretset + ((p2.prorettype != 'index_am_handler'::regtype + AND p2.prorettype != 'compression_am_handler'::regtype) + OR p2.proretset OR p2.pronargs != 1 OR p2.proargtypes[0] != 'internal'::regtype); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 17bf55c1f5..8f85146a62 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -138,6 +138,8 @@ AttoptCacheKey AttrDefInfo AttrDefault AttrNumber +AttributeCompressionInfo +AttributeCompressionItem AttributeOpts AuthRequest AutoPrewarmSharedState @@ -343,6 +345,7 @@ CollectedCommand CollectedCommandType ColorTrgm ColorTrgmInfo +ColumnCompression ColumnCompareData ColumnDef ColumnIOData @@ -368,6 +371,8 @@ CompositeIOData CompositeTypeStmt CompoundAffixFlag CompressionAlgorithm +CompressionAmOptions +CompressionAmRoutine CompressorState ConditionVariable ConditionalStack --MP_/awZ+sOzvK+HW9/=nLSA94yJ--