public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v24 09/10] new compression method extension for zlib 3+ messages / 3 participants [nested] [flat]
* [PATCH v24 09/10] new compression method extension for zlib @ 2020-10-16 14:26 dilipkumar <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: dilipkumar @ 2020-10-16 14:26 UTC (permalink / raw) Dilip Kumar --- contrib/Makefile | 1 + contrib/cmzlib/.gitignore | 4 + contrib/cmzlib/Makefile | 26 +++++ contrib/cmzlib/cmzlib--1.0.sql | 13 +++ contrib/cmzlib/cmzlib.c | 157 +++++++++++++++++++++++++++++ contrib/cmzlib/cmzlib.control | 5 + contrib/cmzlib/expected/cmzlib.out | 53 ++++++++++ contrib/cmzlib/sql/cmzlib.sql | 22 ++++ 8 files changed, 281 insertions(+) create mode 100644 contrib/cmzlib/.gitignore create mode 100644 contrib/cmzlib/Makefile create mode 100644 contrib/cmzlib/cmzlib--1.0.sql create mode 100644 contrib/cmzlib/cmzlib.c create mode 100644 contrib/cmzlib/cmzlib.control create mode 100644 contrib/cmzlib/expected/cmzlib.out create mode 100644 contrib/cmzlib/sql/cmzlib.sql diff --git a/contrib/Makefile b/contrib/Makefile index f27e458482..9e452d8dd0 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -12,6 +12,7 @@ SUBDIRS = \ bloom \ btree_gin \ btree_gist \ + cmzlib \ citext \ cube \ dblink \ diff --git a/contrib/cmzlib/.gitignore b/contrib/cmzlib/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/cmzlib/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/cmzlib/Makefile b/contrib/cmzlib/Makefile new file mode 100644 index 0000000000..956fbe7cc8 --- /dev/null +++ b/contrib/cmzlib/Makefile @@ -0,0 +1,26 @@ +# contrib/cmzlib/Makefile + +MODULE_big = cmzlib +OBJS = \ + $(WIN32RES) \ + cmzlib.o + +EXTENSION = cmzlib +DATA = cmzlib--1.0.sql +PGFILEDESC = "zlib compression method " + +SHLIB_LINK += $(filter -lz, $(LIBS)) + +REGRESS = cmzlib + + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/cmzlib +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/cmzlib/cmzlib--1.0.sql b/contrib/cmzlib/cmzlib--1.0.sql new file mode 100644 index 0000000000..41f2f95870 --- /dev/null +++ b/contrib/cmzlib/cmzlib--1.0.sql @@ -0,0 +1,13 @@ +/* contrib/cm_lz4/cmzlib--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cmzlib" to load this file. \quit + +CREATE FUNCTION zlibhandler(internal) +RETURNS compression_am_handler +AS 'MODULE_PATHNAME' +LANGUAGE C; + +-- Compression method +CREATE ACCESS METHOD zlib TYPE COMPRESSION HANDLER zlibhandler; +COMMENT ON ACCESS METHOD zlib IS 'zlib compression method'; diff --git a/contrib/cmzlib/cmzlib.c b/contrib/cmzlib/cmzlib.c new file mode 100644 index 0000000000..686a7c7e0d --- /dev/null +++ b/contrib/cmzlib/cmzlib.c @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------- + * + * cmzlib.c + * zlib compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * contrib/cmzlib/cmzlib.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/compressamapi.h" +#include "access/toast_internals.h" + +#include "fmgr.h" +#include "utils/builtins.h" + +#include <zlib.h> + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(zlibhandler); + +void _PG_init(void); + +/* + * Module initialize function: initialize info about zlib + */ +void +_PG_init(void) +{ + +} + +#define ZLIB_MAX_DICTIONARY_LENGTH 32768 +#define ZLIB_DICTIONARY_DELIM (" ,") + +typedef struct +{ + int level; + Bytef dict[ZLIB_MAX_DICTIONARY_LENGTH]; + unsigned int dictlen; +} zlib_state; + +/* + * zlib_cmcompress - compression routine for zlib compression method + * + * Compresses source into dest using the default compression level. + * Returns the compressed varlena, or NULL if compression fails. + */ +static struct varlena * +zlib_cmcompress(const struct varlena *value, int32 header_size) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + z_streamp zp; + int res; + zlib_state state; + + state.level = Z_DEFAULT_COMPRESSION; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (deflateInit(zp, state.level) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + tmp = (struct varlena *) palloc(valsize + header_size); + zp->next_in = (void *) VARDATA_ANY(value); + zp->avail_in = valsize; + zp->avail_out = valsize; + zp->next_out = (void *) ((char *) tmp + header_size); + + do + { + res = deflate(zp, Z_FINISH); + if (res == Z_STREAM_ERROR) + elog(ERROR, "could not compress data: %s", zp->msg); + } while (zp->avail_in != 0); + + Assert(res == Z_STREAM_END); + + len = valsize - zp->avail_out; + if (deflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression stream: %s", zp->msg); + pfree(zp); + + if (len > 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + header_size); + return tmp; + } + + pfree(tmp); + return NULL; +} + +/* + * zlib_cmdecompress - decompression routine for zlib compression method + * + * Returns the decompressed varlena. + */ +static struct varlena * +zlib_cmdecompress(const struct varlena *value, int32 header_size) +{ + struct varlena *result; + z_streamp zp; + int res = Z_OK; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (inflateInit(zp) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + zp->next_in = (void *) ((char *) value + header_size); + zp->avail_in = VARSIZE(value) - header_size; + zp->avail_out = VARRAWSIZE_4B_C(value); + + result = (struct varlena *) palloc(zp->avail_out + VARHDRSZ); + SET_VARSIZE(result, zp->avail_out + VARHDRSZ); + zp->next_out = (void *) VARDATA(result); + + while (zp->avail_in > 0) + { + res = inflate(zp, 0); + if (!(res == Z_OK || res == Z_STREAM_END)) + elog(ERROR, "could not uncompress data: %s", zp->msg); + } + + if (inflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression library: %s", zp->msg); + + pfree(zp); + return result; +} + +const CompressionAmRoutine zlib_compress_methods = { + .type = T_CompressionAmRoutine, + .datum_compress = zlib_cmcompress, + .datum_decompress = zlib_cmdecompress, + .datum_decompress_slice = NULL}; + +Datum +zlibhandler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&zlib_compress_methods); +} diff --git a/contrib/cmzlib/cmzlib.control b/contrib/cmzlib/cmzlib.control new file mode 100644 index 0000000000..2eb10f3a83 --- /dev/null +++ b/contrib/cmzlib/cmzlib.control @@ -0,0 +1,5 @@ +# cm_lz4 extension +comment = 'cmzlib compression method' +default_version = '1.0' +module_pathname = '$libdir/cmzlib' +relocatable = true diff --git a/contrib/cmzlib/expected/cmzlib.out b/contrib/cmzlib/expected/cmzlib.out new file mode 100644 index 0000000000..2b6fac7e0b --- /dev/null +++ b/contrib/cmzlib/expected/cmzlib.out @@ -0,0 +1,53 @@ +CREATE EXTENSION cmzlib; +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz); +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + length +-------- + 10040 + 24096 +(2 rows) + +-- alter compression method with rewrite +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz; +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zlib | | + +-- preserve old compression method +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib); +INSERT INTO zlibtest VALUES (repeat('1234567890',1004)); +\d+ zlibtest + Table "public.zlibtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +SELECT pg_column_compression(f1) FROM zlibtest; + pg_column_compression +----------------------- + zlib + zlib + pglz +(3 rows) + +SELECT length(f1) FROM zlibtest; + length +-------- + 10040 + 24096 + 10040 +(3 rows) + +DROP TABLE zlibtest; diff --git a/contrib/cmzlib/sql/cmzlib.sql b/contrib/cmzlib/sql/cmzlib.sql new file mode 100644 index 0000000000..ea8d206625 --- /dev/null +++ b/contrib/cmzlib/sql/cmzlib.sql @@ -0,0 +1,22 @@ +CREATE EXTENSION cmzlib; + +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz); +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + +-- alter compression method with rewrite +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz; +\d+ zlibtest +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ zlibtest + +-- preserve old compression method +ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib); +INSERT INTO zlibtest VALUES (repeat('1234567890',1004)); +\d+ zlibtest +SELECT pg_column_compression(f1) FROM zlibtest; +SELECT length(f1) FROM zlibtest; + +DROP TABLE zlibtest; -- 2.17.0 --m51xatjYGsM+13rf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0010-Support-compression-methods-options.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Order changes in PG16 since ICU introduction @ 2023-05-26 16:24 Daniel Verite <[email protected]> 0 siblings, 1 reply; 3+ messages in thread From: Daniel Verite @ 2023-05-26 16:24 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Andrew Gierth <[email protected]>; Peter Eisentraut <[email protected]>; Sandro Santilli <[email protected]>; Tom Lane <[email protected]>; Regina Obe <[email protected]>; [email protected] Jeff Davis wrote: > > #1 > > > > postgres=# create database test1 locale='fr_FR.UTF-8'; > > NOTICE: using standard form "fr-FR" for ICU locale "fr_FR.UTF-8" > > ERROR: new ICU locale (fr-FR) is incompatible with the ICU locale of > > I don't see a problem here. If you specify LOCALE to CREATE DATABASE, > you should either be using "TEMPLATE template0", or you should be > expecting an error if the LOCALE doesn't match exactly. > > What would you like to see happen here? What's odd is that initdb starting in an fr_FR.UTF-8 environment found that "fr" was the default ICU locale to use, whereas "create database" reports that "fr" and "fr_FR.UTF-8" refer to incompatible locales. To me initdb is wrong when coming up with the less precise "fr" instead of "fr-FR". I suggest the attached patch to call uloc_getDefault() instead of the current code that somehow leaves out the country/region component. Best regards, -- Daniel Vérité https://postgresql.verite.pro/ Twitter: @DanielVerite Attachments: [text/x-patch] initdb-uloc_getdefault.diff (1.3K, ../../[email protected]/2-initdb-uloc_getdefault.diff) download | inline diff: diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 31156e863b..09a5c98cc0 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -2354,42 +2354,13 @@ icu_validate_locale(const char *loc_str) } /* - * Determine default ICU locale by opening the default collator and reading - * its locale. - * - * NB: The default collator (opened using NULL) is different from the collator - * for the root locale (opened with "", "und", or "root"). The former depends - * on the environment (useful at initdb time) and the latter does not. + * Determine the default ICU locale */ static char * default_icu_locale(void) { #ifdef USE_ICU - UCollator *collator; - UErrorCode status; - const char *valid_locale; - char *default_locale; - - status = U_ZERO_ERROR; - collator = ucol_open(NULL, &status); - if (U_FAILURE(status)) - pg_fatal("could not open collator for default locale: %s", - u_errorName(status)); - - status = U_ZERO_ERROR; - valid_locale = ucol_getLocaleByType(collator, ULOC_VALID_LOCALE, - &status); - if (U_FAILURE(status)) - { - ucol_close(collator); - pg_fatal("could not determine default ICU locale"); - } - - default_locale = pg_strdup(valid_locale); - - ucol_close(collator); - - return default_locale; + return pg_strdup(uloc_getDefault()); #else pg_fatal("ICU is not supported in this build"); #endif ^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Order changes in PG16 since ICU introduction @ 2023-05-26 18:47 Jeff Davis <[email protected]> parent: Daniel Verite <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Jeff Davis @ 2023-05-26 18:47 UTC (permalink / raw) To: Daniel Verite <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Andrew Gierth <[email protected]>; Peter Eisentraut <[email protected]>; Sandro Santilli <[email protected]>; Tom Lane <[email protected]>; Regina Obe <[email protected]>; [email protected] On Fri, 2023-05-26 at 18:24 +0200, Daniel Verite wrote: > To me initdb is wrong when coming up with the less precise "fr" > instead of "fr-FR". > > I suggest the attached patch to call uloc_getDefault() instead of the > current code that somehow leaves out the country/region component. Thank you. I experimented with several ICU versions and different environmental settings, and it does seem better at preserving the name that comes from the environment. There is a warning in the docs: "Do not use unless you know what you are doing."[1] I don't see a reason for the warning or any major risk from us using it. Perhaps it's because the result is affected by either the environment or the last uloc_setDefault() call. We don't use uloc_setDefault(), and we only call uloc_getDefault() once, so I don't see a risk here. The fix seems simple enough. Committed. Regards, Jeff Davis [1] https://unicode-org.github.io/icu-docs/apidoc/dev/icu4c/uloc_8h.html#a4efa16db7351e62293f8ef0c37aac8... ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2023-05-26 18:47 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-10-16 14:26 [PATCH v24 09/10] new compression method extension for zlib dilipkumar <[email protected]> 2023-05-26 16:24 Re: Order changes in PG16 since ICU introduction Daniel Verite <[email protected]> 2023-05-26 18:47 ` Re: Order changes in PG16 since ICU introduction Jeff Davis <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox