agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 4/8] Add pglz compression method 7+ messages / 2 participants [nested] [flat]
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index bef53a319a..6f7ad79613 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -30,5 +30,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 02431b45ef..0b2dc896e7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -851,6 +851,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.19.1 --MP_/7ZVSJ3tZdZjf_J65xpltpI_ Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v20.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index bef53a319a..6f7ad79613 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -30,5 +30,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 31768c9fa2..276d2b9b26 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -956,6 +956,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.18.0 --MP_/tqROVSJLfUtKS/DWevR5Hf= Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v19.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index bef53a319a..6f7ad79613 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -30,5 +30,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 31768c9fa2..276d2b9b26 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -956,6 +956,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.18.0 --MP_/tqROVSJLfUtKS/DWevR5Hf= Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v19.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index 393b41dd68..4bf1c49d11 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -33,5 +33,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4cfe44a8da..5ff8e886bd 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -909,6 +909,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.21.0 --MP_//XvyHCr_hJMvh/tp2R3=uJa Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v22.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index 08f331d4e1..adfc10c443 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -30,5 +30,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index be2918be55..5b57d46afe 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -902,6 +902,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.20.1 --MP_/IoARF=uydinlHTRfc3S.uiA Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v21.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 4/8] Add pglz compression method @ 2018-06-18 12:48 Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Ildus Kurbangaliev @ 2018-06-18 12:48 UTC (permalink / raw) Signed-off-by: Ildus Kurbangaliev <i.kurbangaliev@gmail.com> --- src/backend/access/compression/Makefile | 2 +- src/backend/access/compression/cm_pglz.c | 166 ++++++++++++++++++++ src/include/access/cmapi.h | 2 +- src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_attr_compression.dat | 2 +- src/include/catalog/pg_proc.dat | 6 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/backend/access/compression/cm_pglz.c diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile index a09dc787ed..14286920d3 100644 --- a/src/backend/access/compression/Makefile +++ b/src/backend/access/compression/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/access/compression top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = cmapi.o +OBJS = cm_pglz.o cmapi.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000..b693cd09f2 --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 resultlen; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + resultlen = VARRAWSIZE_4B_C(value) + VARHDRSZ; + result = (struct varlena *) palloc(resultlen); + + SET_VARSIZE(result, resultlen); + if (pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value)) < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + + PG_RETURN_POINTER(routine); +} diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h index 9e48f0d49f..1be98a60a5 100644 --- a/src/include/access/cmapi.h +++ b/src/include/access/cmapi.h @@ -19,7 +19,7 @@ #include "nodes/pg_list.h" #define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) -#define DefaultCompressionOid (InvalidOid) +#define DefaultCompressionOid (PGLZ_AC_OID) typedef struct CompressionAmRoutine CompressionAmRoutine; diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index bef53a319a..6f7ad79613 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -30,5 +30,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4002', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat index 30faae0de4..4e72bde16c 100644 --- a/src/include/catalog/pg_attr_compression.dat +++ b/src/include/catalog/pg_attr_compression.dat @@ -18,6 +18,6 @@ [ - +{ acoid => '4002', acname => 'pglz' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2278d87d4d..9f855857e3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -851,6 +851,12 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4009', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, -- 2.19.2 --MP_/P_fMam4EyVB+zJLBGfX5oUD Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0005-Add-zlib-compression-method-v20.patch ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 2/3] one more stress test for repack concurrently @ 2025-12-13 17:46 Mikhail Nikalayeu <mihailnikalayeu@gmail.com> 0 siblings, 0 replies; 7+ messages in thread From: Mikhail Nikalayeu @ 2025-12-13 17:46 UTC (permalink / raw) --- contrib/amcheck/meson.build | 1 + contrib/amcheck/t/008_repack_2.pl | 111 ++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 contrib/amcheck/t/008_repack_2.pl diff --git a/contrib/amcheck/meson.build b/contrib/amcheck/meson.build index cb4bc32e98a..6799df214f9 100644 --- a/contrib/amcheck/meson.build +++ b/contrib/amcheck/meson.build @@ -51,6 +51,7 @@ tests += { 't/005_pitr.pl', 't/006_verify_gin.pl', 't/007_repack_1.pl', + 't/008_repack_2.pl', ], }, } diff --git a/contrib/amcheck/t/008_repack_2.pl b/contrib/amcheck/t/008_repack_2.pl new file mode 100644 index 00000000000..a287ee2bb71 --- /dev/null +++ b/contrib/amcheck/t/008_repack_2.pl @@ -0,0 +1,111 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# Test REPACK CONCURRENTLY with concurrent modifications +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +# PG_TEST_EXTRA carries a numerical stress value; 0 disables the test, +# 1 means the test takes about 6 seconds, and the increase should be +# roughly linear. +my $matched = ($ENV{PG_TEST_EXTRA} // '') =~ /\bstress_concurrently(?:=(?<stressval>[0-9]*))?\b/; +my $stressval = $+{stressval} // 1; +if (!$matched or $stressval == 0) +{ + plan skip_all => 'skipping disabled REPACK CONCURRENTLY stress test'; +} + +my $node; + +# +# Test set-up +# +$node = PostgreSQL::Test::Cluster->new('CIC_test'); +$node->init; +$node->append_conf('postgresql.conf', + 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default)); +$node->append_conf( + 'postgresql.conf', qq( +wal_level = logical +)); + +my $duration = 6 * $stressval; +my $no_hot = int(rand(2)); + +$node->start; +$node->safe_psql('postgres', q(CREATE TABLE tbl(id SERIAL PRIMARY KEY, val int))); +if ($no_hot) +{ + $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(val);)); +} +else +{ + $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(id);)); +} + +# Load amcheck +$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); + +my $sum = $node->safe_psql('postgres', q( + SELECT SUM(val) AS sum FROM tbl +)); + +$node->safe_psql('postgres', q(CREATE UNLOGGED SEQUENCE last_j START 1 INCREMENT 1;)); + + +$node->pgbench( +"--no-vacuum --client=30 --jobs=4 --exit-on-abort -T $duration", +0, +[qr{actually processed}], +[qr{^$}], +'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY', +{ + 'concurrent_ops' => qq( + SELECT pg_try_advisory_lock(42)::integer AS gotlock \\gset + \\if :gotlock + REPACK (CONCURRENTLY) tbl USING INDEX tbl_pkey; + SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true); + SELECT bt_index_parent_check('test_idx', heapallindexed => true); + \\sleep 10 ms + + REPACK (CONCURRENTLY) tbl USING INDEX test_idx; + SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true); + SELECT bt_index_parent_check('test_idx', heapallindexed => true); + \\sleep 10 ms + + REPACK (CONCURRENTLY) tbl; + SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true); + SELECT bt_index_parent_check('test_idx', heapallindexed => true); + \\sleep 10 ms + + SELECT pg_advisory_unlock(42); + \\else + SELECT pg_advisory_lock(43); + BEGIN; + INSERT INTO tbl(val) VALUES (nextval('last_j')) RETURNING val \\gset p_ + COMMIT; + SELECT pg_advisory_unlock(43); + \\sleep 1 ms + + BEGIN + --TRANSACTION ISOLATION LEVEL REPEATABLE READ + ; + SELECT 1; + \\sleep 1 ms + SELECT COUNT(*) AS count FROM tbl WHERE val <= :p_j \\gset p_ + \\if :p_count != :p_j + COMMIT; + SELECT (:p_count) / 0; + \\endif + + COMMIT; + \\endif + ) +}); + +$node->stop; +done_testing(); -- 2.47.3 --tbdehtstrqwksfdh Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="0003-Distinguish-properly-when-database-specific-transact.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2025-12-13 17:46 UTC | newest] Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2018-06-18 12:48 [PATCH 4/8] Add pglz compression method Ildus Kurbangaliev <i.kurbangaliev@gmail.com> 2025-12-13 17:46 [PATCH 2/3] one more stress test for repack concurrently Mikhail Nikalayeu <mihailnikalayeu@gmail.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox