public inbox for [email protected]
help / color / mirror / Atom feedFrom: Ildus Kurbangaliev <[email protected]>
Subject: [PATCH 5/8] Add zlib compression method
Date: Mon, 18 Jun 2018 15:51:07 +0300
Signed-off-by: Ildus Kurbangaliev <[email protected]>
---
src/backend/Makefile | 2 +-
src/backend/access/compression/Makefile | 2 +-
src/backend/access/compression/cm_zlib.c | 252 ++++++++++++++++++++
src/include/catalog/pg_am.dat | 3 +
src/include/catalog/pg_attr_compression.dat | 1 +
src/include/catalog/pg_proc.dat | 4 +
6 files changed, 262 insertions(+), 2 deletions(-)
create mode 100644 src/backend/access/compression/cm_zlib.c
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 478a96db9b..bd5009e190 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -45,7 +45,7 @@ OBJS = $(SUBDIROBJS) $(LOCALOBJS) $(top_builddir)/src/port/libpgport_srv.a \
LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS)
# The backend doesn't need everything that's in LIBS, however
-LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
+LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
ifeq ($(with_systemd),yes)
LIBS += -lsystemd
diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile
index 14286920d3..7ea5ee2e43 100644
--- a/src/backend/access/compression/Makefile
+++ b/src/backend/access/compression/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/access/compression
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
-OBJS = cm_pglz.o cmapi.o
+OBJS = cm_pglz.o cm_zlib.o cmapi.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/compression/cm_zlib.c b/src/backend/access/compression/cm_zlib.c
new file mode 100644
index 0000000000..0dcb56ddf3
--- /dev/null
+++ b/src/backend/access/compression/cm_zlib.c
@@ -0,0 +1,252 @@
+/*-------------------------------------------------------------------------
+ *
+ * cm_zlib.c
+ * zlib compression method
+ *
+ * Copyright (c) 2015-2018, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/access/compression/cm_zlib.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+#include "access/cmapi.h"
+#include "commands/defrem.h"
+#include "nodes/parsenodes.h"
+#include "utils/builtins.h"
+
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+
+#define ZLIB_MAX_DICTIONARY_LENGTH 32768
+#define ZLIB_DICTIONARY_DELIM (" ,")
+
+typedef struct
+{
+ int level;
+ Bytef dict[ZLIB_MAX_DICTIONARY_LENGTH];
+ unsigned int dictlen;
+} zlib_state;
+
+/*
+ * Check options if specified. All validation is located here so
+ * we don't need do it again in cminitstate function.
+ */
+static void
+zlib_cmcheck(Form_pg_attribute att, List *options)
+{
+ ListCell *lc;
+ foreach(lc, options)
+ {
+ DefElem *def = (DefElem *) lfirst(lc);
+
+ if (strcmp(def->defname, "level") == 0)
+ {
+ if (strcmp(defGetString(def), "best_speed") != 0 &&
+ strcmp(defGetString(def), "best_compression") != 0 &&
+ strcmp(defGetString(def), "default") != 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("unexpected value for zlib compression level: \"%s\"", defGetString(def))));
+ }
+ else if (strcmp(def->defname, "dict") == 0)
+ {
+ int ntokens = 0;
+ char *val,
+ *tok;
+
+ val = pstrdup(defGetString(def));
+ if (strlen(val) > (ZLIB_MAX_DICTIONARY_LENGTH - 1))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ (errmsg("zlib dictionary length should be less than %d", ZLIB_MAX_DICTIONARY_LENGTH))));
+
+ while ((tok = strtok(val, ZLIB_DICTIONARY_DELIM)) != NULL)
+ {
+ ntokens++;
+ val = NULL;
+ }
+
+ if (ntokens < 2)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ (errmsg("zlib dictionary is too small"))));
+ }
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_PARAMETER),
+ errmsg("unexpected parameter for zlib: \"%s\"", def->defname)));
+ }
+}
+
+static void *
+zlib_cminitstate(Oid acoid, List *options)
+{
+ zlib_state *state = NULL;
+
+ state = palloc0(sizeof(zlib_state));
+ state->level = Z_DEFAULT_COMPRESSION;
+
+ if (list_length(options) > 0)
+ {
+ ListCell *lc;
+
+ foreach(lc, options)
+ {
+ DefElem *def = (DefElem *) lfirst(lc);
+
+ if (strcmp(def->defname, "level") == 0)
+ {
+ if (strcmp(defGetString(def), "best_speed") == 0)
+ state->level = Z_BEST_SPEED;
+ else if (strcmp(defGetString(def), "best_compression") == 0)
+ state->level = Z_BEST_COMPRESSION;
+ }
+ else if (strcmp(def->defname, "dict") == 0)
+ {
+ char *val,
+ *tok;
+
+ val = pstrdup(defGetString(def));
+
+ /* Fill the zlib dictionary */
+ while ((tok = strtok(val, ZLIB_DICTIONARY_DELIM)) != NULL)
+ {
+ int len = strlen(tok);
+ memcpy((void *) (state->dict + state->dictlen), tok, len);
+ state->dictlen += len + 1;
+ Assert(state->dictlen <= ZLIB_MAX_DICTIONARY_LENGTH);
+
+ /* add space as dictionary delimiter */
+ state->dict[state->dictlen - 1] = ' ';
+ val = NULL;
+ }
+ }
+ }
+ }
+
+ return state;
+}
+
+static struct varlena *
+zlib_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value)
+{
+ int32 valsize,
+ len;
+ struct varlena *tmp = NULL;
+ z_streamp zp;
+ int res;
+ zlib_state *state = (zlib_state *) cmoptions->acstate;
+
+ zp = (z_streamp) palloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ if (deflateInit(zp, state->level) != Z_OK)
+ elog(ERROR, "could not initialize compression library: %s", zp->msg);
+
+ if (state->dictlen > 0)
+ {
+ res = deflateSetDictionary(zp, state->dict, state->dictlen);
+ if (res != Z_OK)
+ elog(ERROR, "could not set dictionary for zlib: %s", zp->msg);
+ }
+
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+ tmp = (struct varlena *) palloc(valsize + VARHDRSZ_CUSTOM_COMPRESSED);
+ zp->next_in = (void *) VARDATA_ANY(value);
+ zp->avail_in = valsize;
+ zp->avail_out = valsize;
+ zp->next_out = (void *)((char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED);
+
+ do {
+ res = deflate(zp, Z_FINISH);
+ if (res == Z_STREAM_ERROR)
+ elog(ERROR, "could not compress data: %s", zp->msg);
+ } while (zp->avail_in != 0);
+
+ Assert(res == Z_STREAM_END);
+
+ len = valsize - zp->avail_out;
+ if (deflateEnd(zp) != Z_OK)
+ elog(ERROR, "could not close compression stream: %s", zp->msg);
+ pfree(zp);
+
+ if (len > 0)
+ {
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED);
+ return tmp;
+ }
+
+ pfree(tmp);
+#endif
+ return NULL;
+}
+
+static struct varlena *
+zlib_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value)
+{
+ struct varlena *result;
+ z_streamp zp;
+ int res = Z_OK;
+ zlib_state *state = (zlib_state *) cmoptions->acstate;
+
+ zp = (z_streamp) palloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ if (inflateInit(zp) != Z_OK)
+ elog(ERROR, "could not initialize compression library: %s", zp->msg);
+
+ Assert(VARATT_IS_CUSTOM_COMPRESSED(value));
+ zp->next_in = (void *) ((char *) value + VARHDRSZ_CUSTOM_COMPRESSED);
+ zp->avail_in = VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED;
+ zp->avail_out = VARRAWSIZE_4B_C(value);
+
+ result = (struct varlena *) palloc(zp->avail_out + VARHDRSZ);
+ SET_VARSIZE(result, zp->avail_out + VARHDRSZ);
+ zp->next_out = (void *) VARDATA(result);
+
+ while (zp->avail_in > 0)
+ {
+ res = inflate(zp, 0);
+ if (res == Z_NEED_DICT && state->dictlen > 0)
+ {
+ res = inflateSetDictionary(zp, state->dict, state->dictlen);
+ if (res != Z_OK)
+ elog(ERROR, "could not set dictionary for zlib");
+ continue;
+ }
+ if (!(res == Z_OK || res == Z_STREAM_END))
+ elog(ERROR, "could not uncompress data: %s", zp->msg);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ elog(ERROR, "could not close compression library: %s", zp->msg);
+
+ pfree(zp);
+ return result;
+}
+
+Datum
+zlibhandler(PG_FUNCTION_ARGS)
+{
+#ifndef HAVE_LIBZ
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("not built with zlib support")));
+#else
+ CompressionAmRoutine *routine = makeNode(CompressionAmRoutine);
+
+ routine->cmcheck = zlib_cmcheck;
+ routine->cminitstate = zlib_cminitstate;
+ routine->cmcompress = zlib_cmcompress;
+ routine->cmdecompress = zlib_cmdecompress;
+
+ PG_RETURN_POINTER(routine);
+#endif
+}
diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat
index adfc10c443..8c101ed157 100644
--- a/src/include/catalog/pg_am.dat
+++ b/src/include/catalog/pg_am.dat
@@ -33,5 +33,8 @@
{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID',
descr => 'pglz compression access method',
amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' },
+{ oid => '4011', oid_symbol => 'ZLIB_COMPRESSION_AM_OID',
+ descr => 'zlib compression access method',
+ amname => 'zlib', amhandler => 'zlibhandler', amtype => 'c' },
]
diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat
index 4e72bde16c..a7216fe963 100644
--- a/src/include/catalog/pg_attr_compression.dat
+++ b/src/include/catalog/pg_attr_compression.dat
@@ -19,5 +19,6 @@
[
{ acoid => '4002', acname => 'pglz' },
+{ acoid => '4011', acname => 'zlib' },
]
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 5b57d46afe..e3dbdb623b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -907,6 +907,10 @@
proname => 'pglzhandler', provolatile => 'v',
prorettype => 'compression_am_handler', proargtypes => 'internal',
prosrc => 'pglzhandler' },
+{ oid => '4010', descr => 'zlib compression access method handler',
+ proname => 'zlibhandler', provolatile => 'v',
+ prorettype => 'compression_am_handler', proargtypes => 'internal',
+ prosrc => 'zlibhandler' },
{ oid => '338', descr => 'validate an operator class',
proname => 'amvalidate', provolatile => 'v', prorettype => 'bool',
--
2.20.1
--MP_/IoARF=uydinlHTRfc3S.uiA
Content-Type: text/x-patch
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename=0006-Add-psql-pg_dump-and-pg_upgrade-support-v21.patch
view thread (23+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH 5/8] Add zlib compression method
In-Reply-To: <no-message-id-1883556@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox