public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v24 09/10] new compression method extension for zlib
3+ messages / 3 participants
[nested] [flat]
* [PATCH v24 09/10] new compression method extension for zlib
@ 2020-10-16 14:26 dilipkumar <[email protected]>
0 siblings, 0 replies; 3+ messages in thread
From: dilipkumar @ 2020-10-16 14:26 UTC (permalink / raw)
Dilip Kumar
---
contrib/Makefile | 1 +
contrib/cmzlib/.gitignore | 4 +
contrib/cmzlib/Makefile | 26 +++++
contrib/cmzlib/cmzlib--1.0.sql | 13 +++
contrib/cmzlib/cmzlib.c | 157 +++++++++++++++++++++++++++++
contrib/cmzlib/cmzlib.control | 5 +
contrib/cmzlib/expected/cmzlib.out | 53 ++++++++++
contrib/cmzlib/sql/cmzlib.sql | 22 ++++
8 files changed, 281 insertions(+)
create mode 100644 contrib/cmzlib/.gitignore
create mode 100644 contrib/cmzlib/Makefile
create mode 100644 contrib/cmzlib/cmzlib--1.0.sql
create mode 100644 contrib/cmzlib/cmzlib.c
create mode 100644 contrib/cmzlib/cmzlib.control
create mode 100644 contrib/cmzlib/expected/cmzlib.out
create mode 100644 contrib/cmzlib/sql/cmzlib.sql
diff --git a/contrib/Makefile b/contrib/Makefile
index f27e458482..9e452d8dd0 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -12,6 +12,7 @@ SUBDIRS = \
bloom \
btree_gin \
btree_gist \
+ cmzlib \
citext \
cube \
dblink \
diff --git a/contrib/cmzlib/.gitignore b/contrib/cmzlib/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/contrib/cmzlib/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/contrib/cmzlib/Makefile b/contrib/cmzlib/Makefile
new file mode 100644
index 0000000000..956fbe7cc8
--- /dev/null
+++ b/contrib/cmzlib/Makefile
@@ -0,0 +1,26 @@
+# contrib/cmzlib/Makefile
+
+MODULE_big = cmzlib
+OBJS = \
+ $(WIN32RES) \
+ cmzlib.o
+
+EXTENSION = cmzlib
+DATA = cmzlib--1.0.sql
+PGFILEDESC = "zlib compression method "
+
+SHLIB_LINK += $(filter -lz, $(LIBS))
+
+REGRESS = cmzlib
+
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/cmzlib
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/cmzlib/cmzlib--1.0.sql b/contrib/cmzlib/cmzlib--1.0.sql
new file mode 100644
index 0000000000..41f2f95870
--- /dev/null
+++ b/contrib/cmzlib/cmzlib--1.0.sql
@@ -0,0 +1,13 @@
+/* contrib/cm_lz4/cmzlib--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION cmzlib" to load this file. \quit
+
+CREATE FUNCTION zlibhandler(internal)
+RETURNS compression_am_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C;
+
+-- Compression method
+CREATE ACCESS METHOD zlib TYPE COMPRESSION HANDLER zlibhandler;
+COMMENT ON ACCESS METHOD zlib IS 'zlib compression method';
diff --git a/contrib/cmzlib/cmzlib.c b/contrib/cmzlib/cmzlib.c
new file mode 100644
index 0000000000..686a7c7e0d
--- /dev/null
+++ b/contrib/cmzlib/cmzlib.c
@@ -0,0 +1,157 @@
+/*-------------------------------------------------------------------------
+ *
+ * cmzlib.c
+ * zlib compression method
+ *
+ * Copyright (c) 2015-2018, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * contrib/cmzlib/cmzlib.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+#include "access/compressamapi.h"
+#include "access/toast_internals.h"
+
+#include "fmgr.h"
+#include "utils/builtins.h"
+
+#include <zlib.h>
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(zlibhandler);
+
+void _PG_init(void);
+
+/*
+ * Module initialize function: initialize info about zlib
+ */
+void
+_PG_init(void)
+{
+
+}
+
+#define ZLIB_MAX_DICTIONARY_LENGTH 32768
+#define ZLIB_DICTIONARY_DELIM (" ,")
+
+typedef struct
+{
+ int level;
+ Bytef dict[ZLIB_MAX_DICTIONARY_LENGTH];
+ unsigned int dictlen;
+} zlib_state;
+
+/*
+ * zlib_cmcompress - compression routine for zlib compression method
+ *
+ * Compresses source into dest using the default compression level.
+ * Returns the compressed varlena, or NULL if compression fails.
+ */
+static struct varlena *
+zlib_cmcompress(const struct varlena *value, int32 header_size)
+{
+ int32 valsize,
+ len;
+ struct varlena *tmp = NULL;
+ z_streamp zp;
+ int res;
+ zlib_state state;
+
+ state.level = Z_DEFAULT_COMPRESSION;
+
+ zp = (z_streamp) palloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ if (deflateInit(zp, state.level) != Z_OK)
+ elog(ERROR, "could not initialize compression library: %s", zp->msg);
+
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+ tmp = (struct varlena *) palloc(valsize + header_size);
+ zp->next_in = (void *) VARDATA_ANY(value);
+ zp->avail_in = valsize;
+ zp->avail_out = valsize;
+ zp->next_out = (void *) ((char *) tmp + header_size);
+
+ do
+ {
+ res = deflate(zp, Z_FINISH);
+ if (res == Z_STREAM_ERROR)
+ elog(ERROR, "could not compress data: %s", zp->msg);
+ } while (zp->avail_in != 0);
+
+ Assert(res == Z_STREAM_END);
+
+ len = valsize - zp->avail_out;
+ if (deflateEnd(zp) != Z_OK)
+ elog(ERROR, "could not close compression stream: %s", zp->msg);
+ pfree(zp);
+
+ if (len > 0)
+ {
+ SET_VARSIZE_COMPRESSED(tmp, len + header_size);
+ return tmp;
+ }
+
+ pfree(tmp);
+ return NULL;
+}
+
+/*
+ * zlib_cmdecompress - decompression routine for zlib compression method
+ *
+ * Returns the decompressed varlena.
+ */
+static struct varlena *
+zlib_cmdecompress(const struct varlena *value, int32 header_size)
+{
+ struct varlena *result;
+ z_streamp zp;
+ int res = Z_OK;
+
+ zp = (z_streamp) palloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ if (inflateInit(zp) != Z_OK)
+ elog(ERROR, "could not initialize compression library: %s", zp->msg);
+
+ zp->next_in = (void *) ((char *) value + header_size);
+ zp->avail_in = VARSIZE(value) - header_size;
+ zp->avail_out = VARRAWSIZE_4B_C(value);
+
+ result = (struct varlena *) palloc(zp->avail_out + VARHDRSZ);
+ SET_VARSIZE(result, zp->avail_out + VARHDRSZ);
+ zp->next_out = (void *) VARDATA(result);
+
+ while (zp->avail_in > 0)
+ {
+ res = inflate(zp, 0);
+ if (!(res == Z_OK || res == Z_STREAM_END))
+ elog(ERROR, "could not uncompress data: %s", zp->msg);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ elog(ERROR, "could not close compression library: %s", zp->msg);
+
+ pfree(zp);
+ return result;
+}
+
+const CompressionAmRoutine zlib_compress_methods = {
+ .type = T_CompressionAmRoutine,
+ .datum_compress = zlib_cmcompress,
+ .datum_decompress = zlib_cmdecompress,
+ .datum_decompress_slice = NULL};
+
+Datum
+zlibhandler(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_POINTER(&zlib_compress_methods);
+}
diff --git a/contrib/cmzlib/cmzlib.control b/contrib/cmzlib/cmzlib.control
new file mode 100644
index 0000000000..2eb10f3a83
--- /dev/null
+++ b/contrib/cmzlib/cmzlib.control
@@ -0,0 +1,5 @@
+# cm_lz4 extension
+comment = 'cmzlib compression method'
+default_version = '1.0'
+module_pathname = '$libdir/cmzlib'
+relocatable = true
diff --git a/contrib/cmzlib/expected/cmzlib.out b/contrib/cmzlib/expected/cmzlib.out
new file mode 100644
index 0000000000..2b6fac7e0b
--- /dev/null
+++ b/contrib/cmzlib/expected/cmzlib.out
@@ -0,0 +1,53 @@
+CREATE EXTENSION cmzlib;
+-- zlib compression
+CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz);
+INSERT INTO zlibtest VALUES(repeat('1234567890',1004));
+INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004));
+SELECT length(f1) FROM zlibtest;
+ length
+--------
+ 10040
+ 24096
+(2 rows)
+
+-- alter compression method with rewrite
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz;
+\d+ zlibtest
+ Table "public.zlibtest"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib;
+\d+ zlibtest
+ Table "public.zlibtest"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zlib | |
+
+-- preserve old compression method
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib);
+INSERT INTO zlibtest VALUES (repeat('1234567890',1004));
+\d+ zlibtest
+ Table "public.zlibtest"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+
+SELECT pg_column_compression(f1) FROM zlibtest;
+ pg_column_compression
+-----------------------
+ zlib
+ zlib
+ pglz
+(3 rows)
+
+SELECT length(f1) FROM zlibtest;
+ length
+--------
+ 10040
+ 24096
+ 10040
+(3 rows)
+
+DROP TABLE zlibtest;
diff --git a/contrib/cmzlib/sql/cmzlib.sql b/contrib/cmzlib/sql/cmzlib.sql
new file mode 100644
index 0000000000..ea8d206625
--- /dev/null
+++ b/contrib/cmzlib/sql/cmzlib.sql
@@ -0,0 +1,22 @@
+CREATE EXTENSION cmzlib;
+
+-- zlib compression
+CREATE TABLE zlibtest(f1 TEXT COMPRESSION pglz);
+INSERT INTO zlibtest VALUES(repeat('1234567890',1004));
+INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004));
+SELECT length(f1) FROM zlibtest;
+
+-- alter compression method with rewrite
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz;
+\d+ zlibtest
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION zlib;
+\d+ zlibtest
+
+-- preserve old compression method
+ALTER TABLE zlibtest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (zlib);
+INSERT INTO zlibtest VALUES (repeat('1234567890',1004));
+\d+ zlibtest
+SELECT pg_column_compression(f1) FROM zlibtest;
+SELECT length(f1) FROM zlibtest;
+
+DROP TABLE zlibtest;
--
2.17.0
--m51xatjYGsM+13rf
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v24-0010-Support-compression-methods-options.patch"
^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Order changes in PG16 since ICU introduction
@ 2023-05-26 16:24 Daniel Verite <[email protected]>
2023-05-26 18:47 ` Re: Order changes in PG16 since ICU introduction Jeff Davis <[email protected]>
0 siblings, 1 reply; 3+ messages in thread
From: Daniel Verite @ 2023-05-26 16:24 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Andrew Gierth <[email protected]>; Peter Eisentraut <[email protected]>; Sandro Santilli <[email protected]>; Tom Lane <[email protected]>; Regina Obe <[email protected]>; [email protected]
Jeff Davis wrote:
> > #1
> >
> > postgres=# create database test1 locale='fr_FR.UTF-8';
> > NOTICE: using standard form "fr-FR" for ICU locale "fr_FR.UTF-8"
> > ERROR: new ICU locale (fr-FR) is incompatible with the ICU locale of
>
> I don't see a problem here. If you specify LOCALE to CREATE DATABASE,
> you should either be using "TEMPLATE template0", or you should be
> expecting an error if the LOCALE doesn't match exactly.
>
> What would you like to see happen here?
What's odd is that initdb starting in an fr_FR.UTF-8 environment
found that "fr" was the default ICU locale to use, whereas
"create database" reports that "fr" and "fr_FR.UTF-8" refer to
incompatible locales.
To me initdb is wrong when coming up with the less precise "fr"
instead of "fr-FR".
I suggest the attached patch to call uloc_getDefault() instead of the
current code that somehow leaves out the country/region component.
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
Twitter: @DanielVerite
Attachments:
[text/x-patch] initdb-uloc_getdefault.diff (1.3K, ../../[email protected]/2-initdb-uloc_getdefault.diff)
download | inline diff:
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 31156e863b..09a5c98cc0 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -2354,42 +2354,13 @@ icu_validate_locale(const char *loc_str)
}
/*
- * Determine default ICU locale by opening the default collator and reading
- * its locale.
- *
- * NB: The default collator (opened using NULL) is different from the collator
- * for the root locale (opened with "", "und", or "root"). The former depends
- * on the environment (useful at initdb time) and the latter does not.
+ * Determine the default ICU locale
*/
static char *
default_icu_locale(void)
{
#ifdef USE_ICU
- UCollator *collator;
- UErrorCode status;
- const char *valid_locale;
- char *default_locale;
-
- status = U_ZERO_ERROR;
- collator = ucol_open(NULL, &status);
- if (U_FAILURE(status))
- pg_fatal("could not open collator for default locale: %s",
- u_errorName(status));
-
- status = U_ZERO_ERROR;
- valid_locale = ucol_getLocaleByType(collator, ULOC_VALID_LOCALE,
- &status);
- if (U_FAILURE(status))
- {
- ucol_close(collator);
- pg_fatal("could not determine default ICU locale");
- }
-
- default_locale = pg_strdup(valid_locale);
-
- ucol_close(collator);
-
- return default_locale;
+ return pg_strdup(uloc_getDefault());
#else
pg_fatal("ICU is not supported in this build");
#endif
^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Order changes in PG16 since ICU introduction
2023-05-26 16:24 Re: Order changes in PG16 since ICU introduction Daniel Verite <[email protected]>
@ 2023-05-26 18:47 ` Jeff Davis <[email protected]>
0 siblings, 0 replies; 3+ messages in thread
From: Jeff Davis @ 2023-05-26 18:47 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Andrew Gierth <[email protected]>; Peter Eisentraut <[email protected]>; Sandro Santilli <[email protected]>; Tom Lane <[email protected]>; Regina Obe <[email protected]>; [email protected]
On Fri, 2023-05-26 at 18:24 +0200, Daniel Verite wrote:
> To me initdb is wrong when coming up with the less precise "fr"
> instead of "fr-FR".
>
> I suggest the attached patch to call uloc_getDefault() instead of the
> current code that somehow leaves out the country/region component.
Thank you. I experimented with several ICU versions and different
environmental settings, and it does seem better at preserving the name
that comes from the environment.
There is a warning in the docs: "Do not use unless you know what you
are doing."[1] I don't see a reason for the warning or any major risk
from us using it. Perhaps it's because the result is affected by either
the environment or the last uloc_setDefault() call. We don't use
uloc_setDefault(), and we only call uloc_getDefault() once, so I don't
see a risk here.
The fix seems simple enough. Committed.
Regards,
Jeff Davis
[1]
https://unicode-org.github.io/icu-docs/apidoc/dev/icu4c/uloc_8h.html#a4efa16db7351e62293f8ef0c37aac8...
^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2023-05-26 18:47 UTC | newest]
Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-10-16 14:26 [PATCH v24 09/10] new compression method extension for zlib dilipkumar <[email protected]>
2023-05-26 16:24 Re: Order changes in PG16 since ICU introduction Daniel Verite <[email protected]>
2023-05-26 18:47 ` Re: Order changes in PG16 since ICU introduction Jeff Davis <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox