public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v24 09/10] new compression method extension for zlib 2+ messages / 2 participants [nested] [flat]
* [PATCH v24 09/10] new compression method extension for zlib @ 2020-10-16 14:26 dilipkumar <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: dilipkumar @ 2020-10-16 14:26 UTC (permalink / raw) Dilip Kumar --- contrib/Makefile | 1 + contrib/cmzlib/.gitignore | 4 + contrib/cmzlib/Makefile | 26 +++++ contrib/cmzlib/cmzlib--1.0.sql | 13 +++ contrib/cmzlib/cmzlib.c | 157 +++++++++++++++++++++++++++++ contrib/cmzlib/cmzlib.control | 5 + contrib/cmzlib/expected/cmzlib.out | 53 ++++++++++ contrib/cmzlib/sql/cmzlib.sql | 22 ++++ 8 files changed, 281 insertions(+) create mode 100644 contrib/cmzlib/.gitignore create mode 100644 contrib/cmzlib/Makefile create mode 100644 contrib/cmzlib/cmzlib--1.0.sql create mode 100644 contrib/cmzlib/cmzlib.c create mode 100644 contrib/cmzlib/cmzlib.control create mode 100644 contrib/cmzlib/expected/cmzlib.out create mode 100644 contrib/cmzlib/sql/cmzlib.sql diff --git a/contrib/Makefile b/contrib/Makefile index f27e458482..9e452d8dd0 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -12,6 +12,7 @@ SUBDIRS = \ bloom \ btree_gin \ btree_gist \ + cmzlib \ citext \ cube \ dblink \ diff --git a/contrib/cmzlib/.gitignore b/contrib/cmzlib/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/cmzlib/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/cmzlib/Makefile b/contrib/cmzlib/Makefile new file mode 100644 index 0000000000..956fbe7cc8 --- /dev/null +++ b/contrib/cmzlib/Makefile @@ -0,0 +1,26 @@ +# contrib/cmzlib/Makefile + +MODULE_big = cmzlib +OBJS = \ + $(WIN32RES) \ + cmzlib.o + +EXTENSION = cmzlib +DATA = cmzlib--1.0.sql +PGFILEDESC = "zlib compression method " + +SHLIB_LINK += $(filter -lz, $(LIBS)) + +REGRESS = cmzlib + + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/cmzlib +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/cmzlib/cmzlib--1.0.sql b/contrib/cmzlib/cmzlib--1.0.sql new file mode 100644 index 0000000000..41f2f95870 --- /dev/null +++ b/contrib/cmzlib/cmzlib--1.0.sql @@ -0,0 +1,13 @@ +/* contrib/cm_lz4/cmzlib--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cmzlib" to load this file. \quit + +CREATE FUNCTION zlibhandler(internal) +RETURNS compression_am_handler +AS 'MODULE_PATHNAME' +LANGUAGE C; + +-- Compression method +CREATE ACCESS METHOD zlib TYPE COMPRESSION HANDLER zlibhandler; +COMMENT ON ACCESS METHOD zlib IS 'zlib compression method'; diff --git a/contrib/cmzlib/cmzlib.c b/contrib/cmzlib/cmzlib.c new file mode 100644 index 0000000000..686a7c7e0d --- /dev/null +++ b/contrib/cmzlib/cmzlib.c @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------- + * + * cmzlib.c + * zlib compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * contrib/cmzlib/cmzlib.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/compressamapi.h" +#include "access/toast_internals.h" + +#include "fmgr.h" +#include "utils/builtins.h" + +#include <zlib.h> + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(zlibhandler); + +void _PG_init(void); + +/* + * Module initialize function: initialize info about zlib + */ +void +_PG_init(void) +{ + +} + +#define ZLIB_MAX_DICTIONARY_LENGTH 32768 +#define ZLIB_DICTIONARY_DELIM (" ,") + +typedef struct +{ + int level; + Bytef dict[ZLIB_MAX_DICTIONARY_LENGTH]; + unsigned int dictlen; +} zlib_state; + +/* + * zlib_cmcompress - compression routine for zlib compression method + * + * Compresses source into dest using the default compression level. + * Returns the compressed varlena, or NULL if compression fails. + */ +static struct varlena * +zlib_cmcompress(const struct varlena *value, int32 header_size) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + z_streamp zp; + int res; + zlib_state state; + + state.level = Z_DEFAULT_COMPRESSION; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (deflateInit(zp, state.level) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + tmp = (struct varlena *) palloc(valsize + header_size); + zp->next_in = (void *) VARDATA_ANY(value); + zp->avail_in = valsize; + zp->avail_out = valsize; + zp->next_out = (void *) ((char *) tmp + header_size); + + do + { + res = deflate(zp, Z_FINISH); + if (res == Z_STREAM_ERROR) + elog(ERROR, "could not compress data: %s", zp->msg); + } while (zp->avail_in != 0); + + Assert(res == Z_STREAM_END); + + len = valsize - zp->avail_out; + if (deflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression stream: %s", zp->msg); + pfree(zp); + + if (len > 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + header_size); + return tmp; + } + + pfree(tmp); + return NULL; +} + +/* + * zlib_cmdecompress - decompression routine for zlib compression method + * + * Returns the decompressed varlena. + */ +static struct varlena * +zlib_cmdecompress(const struct varlena *value, int32 header_size) +{ + struct varlena *result; + z_streamp zp; + int res = Z_OK; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (inflateInit(zp) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + zp->next_in = (void *) ((char *) value + header_size); + zp->avail_in = VARSIZE(value) - header_size; + zp->avail_out = VARRAWSIZE_4B_C(value); + + result = (struct varlena *) palloc(zp->avail_out + VARHDRSZ); + SET_VARSIZE(result, zp->avail_out + VARHDRSZ); + zp->next_out = (void *) VARDATA(result); + + while (zp->avail_in > 0) + { + res = inflate(zp, 0); + if (!(res == Z_OK || res == Z_STREAM_END)) + elog(ERROR, "could not uncompress data: %s", zp->msg); + } + + if (inflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression library: %s", zp->msg); + + pfree(zp); + return result; +} + +const CompressionAmRoutine zlib_compress_methods = { + .type = T_CompressionAmRoutine, + .datum_compress = zlib_cmcompress, + .datum_decompress = zlib_cmdecompress, + .datum_decompress_slice = NULL}; + +Datum +zlibhandler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&zlib_compress_methods); +} diff --git a/contrib/cmzlib/cmzlib.control b/contrib/cmzlib/cmzlib.control new file mode 100644 index 0000000000..2eb10f3a83 --- /dev/null +++ b/contrib/cmzlib/cmzlib.control @@ -0,0 +1,5 @@ +# cm_lz4 extension +comment = 'cmzlib compression method' +default_version = '1.0' +module_pathname = '$libdir/cmzlib' +relocatable = true diff --git a/contrib/cmzlib/expected/cmzlib.out b/contrib/cmzlib/expected/cmzlib.out new file mode 100644 index 0000000000..2b6fac7e0b --- /dev/null +++ b/contrib/cmzlib/expected/cmzlib.out @@ -0,0 +1,53 @@ +CREATE EXTENSION cmzlib; +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz); +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + length +-------- + 10040 + 24096 +(2 rows) + +-- alter compression method with rewrite +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz; +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zlib | | + +-- preserve old compression method +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib); +INSERT INTO zlibtest VALUES (repeat('1234567890',1004)); +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +SELECT pg_column_compression(f1) FROM zlibtest; + pg_column_compression +----------------------- + zlib + zlib + pglz +(3 rows) + +SELECT length(f1) FROM zlibtest; + length +-------- + 10040 + 24096 + 10040 +(3 rows) + +DROP TABLE zlibtest; diff --git a/contrib/cmzlib/sql/cmzlib.sql b/contrib/cmzlib/sql/cmzlib.sql new file mode 100644 index 0000000000..ea8d206625 --- /dev/null +++ b/contrib/cmzlib/sql/cmzlib.sql @@ -0,0 +1,22 @@ +CREATE EXTENSION cmzlib; + +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz); +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + +-- alter compression method with rewrite +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz; +\d+ zlibtest +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ zlibtest + +-- preserve old compression method +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib); +INSERT INTO zlibtest VALUES (repeat('1234567890',1004)); +\d+ zlibtest +SELECT pg_column_compression(f1) FROM zlibtest; +SELECT length(f1) FROM zlibtest; + +DROP TABLE zlibtest; -- 2.17.0 --m51xatjYGsM+13rf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0010-Support-compression-methods-options.patch" ^ permalink raw reply [nested|flat] 2+ messages in thread
* Remove extra Sort node above a btree-compatible Index Scan @ 2025-02-28 05:22 Alexandra Wang <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Alexandra Wang @ 2025-02-28 05:22 UTC (permalink / raw) To: pgsql-hackers Hello hackers, While reviewing Mark Dilger’s patch series "Index AM API cleanup" [1], I noticed some unexpected plan diffs when running the xtree and treeb test modules. Specifically, an additional Sort node appears on top of an Index Only Scan, even when the index key and sort key are the same. For example: --- src/test/modules/xtree/expected/interval.out +++ src/test/modules/xtree/results/interval.out @@ -375,10 +375,12 @@ SET enable_seqscan TO false; EXPLAIN (COSTS OFF) SELECT f1 FROM INTERVAL_TBL_OF r1 ORDER BY f1; - QUERY PLAN --------------------------------------------------------------------- - Index Only Scan using interval_tbl_of_f1_idx on interval_tbl_of r1 -(1 row) + QUERY PLAN +-------------------------------------------------------------------------- + Sort + Sort Key: f1 + -> Index Only Scan using interval_tbl_of_f1_idx on interval_tbl_of r1 +(3 rows) I’ve attached a patch that removes this unnecessary Sort node for B-tree-compatible indexes. This change should: - Reduce the number of test failures in the xtree module from 43 to 30 - Reduce the size of regression.diffs from 234K to 123K Since pathkey comparison is a hot path in query planning and exercised by many test queries, I plan to gather more performance metrics. However, in a simple test running make installcheck with and without the patch, I observed no negative impact on the runtime of the regression test suite (which doesn’t include other btree-like indexes) and a positive impact on the same regression tests for xtree. Regression tests (same plans): w/o patch: make installcheck 1.36s user 2.21s system 12% cpu 28.018 total w/ patch: make installcheck 1.32s user 2.12s system 12% cpu 28.089 total xtree tests: w/o patch (inferior plan w/ extra sort node): make installcheck 1.52s user 2.42s system 10% cpu 36.817 total w/ patch (good plan): make installcheck 1.52s user 2.48s system 12% cpu 32.201 total I’ve marked the patch as no-cfbot, as it applies only on top of the aforementioned patch series [1]. Thoughts? [1] https://www.postgresql.org/message-id/a5dfb7cd-7a89-48ab-a913-e5304eee0854%40eisentraut.org Best, Alex Attachments: [application/octet-stream] v1-0001-Remove-unnecessary-Sort-node-above-Index-Scan-of-.patch.no-cfbot (5.6K, ../../CAK98qZ0MVVdV1tQKhHq=2qVkKN1rWff-YW6BGOOLoO509krv4Q@mail.gmail.com/3-v1-0001-Remove-unnecessary-Sort-node-above-Index-Scan-of-.patch.no-cfbot) download ^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2025-02-28 05:22 UTC | newest] Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-10-16 14:26 [PATCH v24 09/10] new compression method extension for zlib dilipkumar <[email protected]> 2025-02-28 05:22 Remove extra Sort node above a btree-compatible Index Scan Alexandra Wang <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox