public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/1] resownerbench
7+ messages / 4 participants
[nested] [flat]
* [PATCH 1/1] resownerbench
@ 2020-11-11 22:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Heikki Linnakangas @ 2020-11-11 22:40 UTC (permalink / raw)
---
contrib/resownerbench/Makefile | 17 ++
contrib/resownerbench/resownerbench--1.0.sql | 14 ++
contrib/resownerbench/resownerbench.c | 154 +++++++++++++++++++
contrib/resownerbench/resownerbench.control | 6 +
contrib/resownerbench/snaptest.sql | 37 +++++
5 files changed, 228 insertions(+)
create mode 100644 contrib/resownerbench/Makefile
create mode 100644 contrib/resownerbench/resownerbench--1.0.sql
create mode 100644 contrib/resownerbench/resownerbench.c
create mode 100644 contrib/resownerbench/resownerbench.control
create mode 100644 contrib/resownerbench/snaptest.sql
diff --git a/contrib/resownerbench/Makefile b/contrib/resownerbench/Makefile
new file mode 100644
index 00000000000..9b0e1cfee1a
--- /dev/null
+++ b/contrib/resownerbench/Makefile
@@ -0,0 +1,17 @@
+MODULE_big = resownerbench
+OBJS = resownerbench.o
+
+EXTENSION = resownerbench
+DATA = resownerbench--1.0.sql
+PGFILEDESC = "resownerbench - benchmark for ResourceOwners"
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/resownerbench
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/resownerbench/resownerbench--1.0.sql b/contrib/resownerbench/resownerbench--1.0.sql
new file mode 100644
index 00000000000..d29182f5982
--- /dev/null
+++ b/contrib/resownerbench/resownerbench--1.0.sql
@@ -0,0 +1,14 @@
+/* contrib/resownerbench/resownerbench--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION resownerbench" to load this file. \quit
+
+CREATE FUNCTION snapshotbench_lifo(numkeep int, numsnaps int, numiters int)
+RETURNS double precision
+AS 'MODULE_PATHNAME', 'snapshotbench_lifo'
+LANGUAGE C STRICT VOLATILE;
+
+CREATE FUNCTION snapshotbench_fifo(numkeep int, numsnaps int, numiters int)
+RETURNS double precision
+AS 'MODULE_PATHNAME', 'snapshotbench_fifo'
+LANGUAGE C STRICT VOLATILE;
diff --git a/contrib/resownerbench/resownerbench.c b/contrib/resownerbench/resownerbench.c
new file mode 100644
index 00000000000..acfb6c39199
--- /dev/null
+++ b/contrib/resownerbench/resownerbench.c
@@ -0,0 +1,154 @@
+#include "postgres.h"
+
+#include "catalog/pg_type.h"
+#include "catalog/pg_statistic.h"
+#include "executor/spi.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "utils/catcache.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/snapmgr.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(snapshotbench_lifo);
+PG_FUNCTION_INFO_V1(snapshotbench_fifo);
+
+/*
+ * ResourceOwner Performance test, using RegisterSnapshot().
+ *
+ * This takes three parameters: numkeep, numsnaps, numiters.
+ *
+ * First, we register 'numkeep' snapshots. They are kept registed
+ * until the end of the test. Then, we repeatedly register and
+ * unregister 'numsnaps - numkeep' additional snapshots, repeating
+ * 'numiters' times. All the register/unregister calls are made in
+ * LIFO order.
+ *
+ * Returns the time spent, in milliseconds.
+ *
+ * The idea is to test the performance of ResourceOwnerRemember()
+ * and ReourceOwnerForget() operations, under different regimes.
+ *
+ * In the old implementation, if 'numsnaps' is small enough, all
+ * the entries fit in the resource owner's small array (it can
+ * hold 64 entries).
+ *
+ * In the new implementation, the array is much smaller, only 8
+ * entries, but it's used together with the hash table so that
+ * we stay in the "array regime" as long as 'numsnaps - numkeep'
+ * is smaller than 8 entries.
+ *
+ * 'numiters' can be adjusted to adjust the overall runtime to be
+ * suitable long.
+ */
+Datum
+snapshotbench_lifo(PG_FUNCTION_ARGS)
+{
+ int numkeep = PG_GETARG_INT32(0);
+ int numsnaps = PG_GETARG_INT32(1);
+ int numiters = PG_GETARG_INT32(2);
+ int i;
+ instr_time start,
+ duration;
+ Snapshot lsnap;
+ Snapshot *rs;
+ int numregistered = 0;
+
+ rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot));
+
+ lsnap = GetLatestSnapshot();
+
+ PG_SETMASK(&BlockSig);
+ INSTR_TIME_SET_CURRENT(start);
+
+ while (numregistered < numkeep)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (i = 0 ; i < numiters; i++)
+ {
+ while (numregistered < numsnaps)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ while (numregistered > numkeep)
+ {
+ numregistered--;
+ UnregisterSnapshot(rs[numregistered]);
+ }
+ }
+
+ while (numregistered > 0)
+ {
+ numregistered--;
+ UnregisterSnapshot(rs[numregistered]);
+ }
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ PG_SETMASK(&UnBlockSig);
+
+ PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration));
+};
+
+
+/*
+ * Same, but do the register/unregister operations in
+ * FIFO order.
+ */
+Datum
+snapshotbench_fifo(PG_FUNCTION_ARGS)
+{
+ int numkeep = PG_GETARG_INT32(0);
+ int numsnaps = PG_GETARG_INT32(1);
+ int numiters = PG_GETARG_INT32(2);
+ int i,
+ j;
+ instr_time start,
+ duration;
+ Snapshot lsnap;
+ Snapshot *rs;
+ int numregistered = 0;
+
+ rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot));
+
+ lsnap = GetLatestSnapshot();
+
+ PG_SETMASK(&BlockSig);
+ INSTR_TIME_SET_CURRENT(start);
+
+ while (numregistered < numkeep)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (i = 0 ; i < numiters; i++)
+ {
+ while (numregistered < numsnaps)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (j = numkeep; j < numregistered; j++)
+ UnregisterSnapshot(rs[j]);
+ numregistered = numkeep;
+ }
+
+ for (j = 0; j < numregistered; j++)
+ UnregisterSnapshot(rs[j]);
+ numregistered = numkeep;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ PG_SETMASK(&UnBlockSig);
+
+ PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration));
+};
diff --git a/contrib/resownerbench/resownerbench.control b/contrib/resownerbench/resownerbench.control
new file mode 100644
index 00000000000..ada88b8eed8
--- /dev/null
+++ b/contrib/resownerbench/resownerbench.control
@@ -0,0 +1,6 @@
+# resownerbench
+
+comment = 'benchmark for ResourceOwners'
+default_version = '1.0'
+module_pathname = '$libdir/resownerbench'
+relocatable = true
diff --git a/contrib/resownerbench/snaptest.sql b/contrib/resownerbench/snaptest.sql
new file mode 100644
index 00000000000..18c54e13fc9
--- /dev/null
+++ b/contrib/resownerbench/snaptest.sql
@@ -0,0 +1,37 @@
+--
+-- Performance test RegisterSnapshot/UnregisterSnapshot.
+--
+select numkeep, numsnaps,
+ -- numiters,
+ -- round(lifo_time_ms) as lifo_total_time_ms,
+ -- round(fifo_time_ms) as fifo_total_time_ms,
+ round((lifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as lifo_time_ns,
+ round((fifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as fifo_time_ns
+from
+(values (0, 1, 10000000),
+ (0, 5, 2000000),
+ (0, 10, 1000000),
+ (0, 60, 100000),
+ (0, 70, 100000),
+ (0, 100, 100000),
+ (0, 1000, 10000),
+ (0, 10000, 1000),
+
+-- These tests keep 9 snapshots registered across the iterations. That
+-- exceeds the size of the little array in the patch, so this exercises
+-- the hash lookups. Without the patch, these still fit in the array
+-- (it's 64 entries without the patch)
+ (9, 10, 10000000),
+ (9, 100, 100000),
+ (9, 1000, 10000),
+ (9, 10000, 1000),
+
+-- These exceed the 64 entry array even without the patch, so these fall
+-- in the hash table regime with and without the patch.
+ (65, 70, 1000000),
+ (65, 100, 100000),
+ (65, 1000, 10000),
+ (65, 10000, 1000)
+) AS params (numkeep, numsnaps, numiters),
+lateral snapshotbench_lifo(numkeep, numsnaps, numiters) as lifo_time_ms,
+lateral snapshotbench_fifo(numkeep, numsnaps, numiters) as fifo_time_ms;
--
2.29.2
--------------1104342304EC6341F5D7C60C--
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH] Compression dictionaries for JSONB
@ 2022-04-22 08:30 Aleksander Alekseev <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Aleksander Alekseev @ 2022-04-22 08:30 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
Hi hackers,
This is a follow-up thread to `RFC: compression dictionaries for JSONB`
[1]. I would like to share my current progress in order to get early
feedback. The patch is currently in a draft state but implements the basic
functionality. I did my best to account for all the great feedback I
previously got from Alvaro and Matthias.
Usage example:
```
CREATE TYPE mydict AS DICTIONARY OF jsonb ('aaa', 'bbb');
SELECT '{"aaa":"bbb"}' :: mydict;
mydict
----------------
{"aaa": "bbb"}
SELECT ('{"aaa":"bbb"}' :: mydict) -> 'aaa';
?column?
----------
"bbb"
```
Here `mydict` works as a transparent replacement for `jsonb`. However, its
internal representation differs. The provided dictionary entries ('aaa',
'bbb') are stored in the new catalog table:
```
SELECT * FROM pg_dict;
oid | dicttypid | dictentry
-------+-----------+-----------
39476 | 39475 | aaa
39477 | 39475 | bbb
(2 rows)
```
When `mydict` sees 'aaa' in the document, it replaces it with the
corresponding code, in this case - 39476. For more details regarding the
compression algorithm and choosen compromises please see the comments in
the patch.
In pg_type `mydict` has typtype = TYPTYPE_DICT. It works the same way as
TYPTYPE_BASE with only difference: corresponding `<type>_in`
(pg_type.typinput) and `<another-type>_<type>` (pg_cast.castfunc)
procedures receive the dictionary Oid as a `typmod` argument. This way the
procedures can distinguish `mydict1` from `mydict2` and use the proper
compression dictionary.
The approach with alternative `typmod` role is arguably a bit hacky, but it
was the less invasive way to implement the feature I've found. I'm open to
alternative suggestions.
Current limitations (todo):
- ALTER TYPE is not implemented
- Tests and documentation are missing
- Autocomplete is missing
Future work (out of scope of this patch):
- Support types other than JSONB: TEXT, XML, etc
- Automatically updated dictionaries, e.g. during VACUUM
- Alternative compression algorithms. Note that this will not require any
further changes in the catalog, only the values we write to pg_type and
pg_cast will differ.
Open questions:
- Dictionary entries are currently stored as NameData, the same type that
is used for enums. Are we OK with the accompanying limitations? Any
alternative suggestions?
- All in all, am I moving the right direction?
Your feedback is very much welcomed!
[1]:
https://postgr.es/m/CAJ7c6TPx7N-bVw0dZ1ASCDQKZJHhBYkT6w4HV1LzfS%2BUUTUfmA%40mail.gmail.com
--
Best regards,
Aleksander Alekseev
Attachments:
[application/octet-stream] v1-0001-CREATE-TYPE-foo-AS-DICTIONARY-OF-JSONB.patch (45.7K, ../../CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22=5xVBg7S4vr5rQ@mail.gmail.com/3-v1-0001-CREATE-TYPE-foo-AS-DICTIONARY-OF-JSONB.patch)
download | inline diff:
From e3832559a2288f3f05f4144a525cf46ccd8a0252 Mon Sep 17 00:00:00 2001
From: Aleksander Alekseev <[email protected]>
Date: Tue, 12 Apr 2022 15:58:11 +0300
Subject: [PATCH v1] CREATE TYPE foo AS DICTIONARY OF JSONB
FIXME explain what the feature is all about and give some
implementation details
(NOTE: dear committer, please bump the catalog version!)
Author: Aleksander Alekseev <[email protected]>
Reviewed-by: FIXME
Discussion: FIXME
---
src/backend/catalog/Makefile | 3 +-
src/backend/catalog/pg_dict.c | 153 ++++++
src/backend/commands/typecmds.c | 151 +++++-
src/backend/executor/functions.c | 1 +
src/backend/nodes/copyfuncs.c | 15 +
src/backend/nodes/equalfuncs.c | 13 +
src/backend/parser/gram.y | 8 +
src/backend/parser/parse_coerce.c | 17 +-
src/backend/parser/parse_type.c | 16 +
src/backend/tcop/utility.c | 13 +
src/backend/utils/adt/Makefile | 1 +
src/backend/utils/adt/dictionaries.c | 476 ++++++++++++++++++
src/backend/utils/fmgr/funcapi.c | 1 +
src/bin/pg_dump/pg_dump.c | 2 +
src/include/catalog/pg_dict.h | 76 +++
src/include/catalog/pg_proc.dat | 20 +
src/include/catalog/pg_type.h | 1 +
src/include/commands/typecmds.h | 1 +
src/include/nodes/nodes.h | 1 +
src/include/nodes/parsenodes.h | 12 +
src/pl/plpgsql/src/pl_comp.c | 1 +
.../modules/test_oat_hooks/test_oat_hooks.c | 1 +
src/test/regress/expected/oidjoins.out | 1 +
src/test/regress/expected/opr_sanity.out | 14 +-
24 files changed, 988 insertions(+), 10 deletions(-)
create mode 100644 src/backend/catalog/pg_dict.c
create mode 100644 src/backend/utils/adt/dictionaries.c
create mode 100644 src/include/catalog/pg_dict.h
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index 89a0221ec9..3b4330148e 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -33,6 +33,7 @@ OBJS = \
pg_conversion.o \
pg_db_role_setting.o \
pg_depend.o \
+ pg_dict.o \
pg_enum.o \
pg_inherits.o \
pg_largeobject.o \
@@ -61,7 +62,7 @@ CATALOG_HEADERS := \
pg_language.h pg_largeobject_metadata.h pg_largeobject.h pg_aggregate.h \
pg_statistic.h pg_statistic_ext.h pg_statistic_ext_data.h \
pg_rewrite.h pg_trigger.h pg_event_trigger.h pg_description.h \
- pg_cast.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \
+ pg_cast.h pg_dict.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \
pg_database.h pg_db_role_setting.h pg_tablespace.h \
pg_authid.h pg_auth_members.h pg_shdepend.h pg_shdescription.h \
pg_ts_config.h pg_ts_config_map.h pg_ts_dict.h \
diff --git a/src/backend/catalog/pg_dict.c b/src/backend/catalog/pg_dict.c
new file mode 100644
index 0000000000..ad10bee442
--- /dev/null
+++ b/src/backend/catalog/pg_dict.c
@@ -0,0 +1,153 @@
+#include "postgres.h"
+
+#include "access/htup_details.h"
+#include "access/genam.h"
+#include "access/skey.h"
+#include "access/table.h"
+#include "catalog/catalog.h"
+#include "catalog/indexing.h"
+#include "catalog/pg_dict.h"
+#include "catalog/pg_type.h"
+#include "utils/builtins.h"
+#include "utils/fmgroids.h"
+#include "utils/rel.h"
+
+/*
+ * Creates an entry in pg_dict for each of the supplied values.
+ * vals is a list of String values.
+ */
+void
+DictEntriesCreate(Oid dictTypeOid, List *vals)
+{
+ Relation pg_dict;
+ NameData dictentry;
+ Datum values[Natts_pg_dict];
+ bool nulls[Natts_pg_dict];
+ ListCell *lc;
+ HeapTuple tup;
+
+ if (vals == NIL)
+ {
+ /* The list is empty; do nothing. */
+ return;
+ }
+
+ memset(nulls, false, sizeof(nulls));
+
+ /*
+ * We don't check the list of values for duplicates here. If there are
+ * any, the user will get an unique-index violation.
+ */
+
+ pg_dict = table_open(DictRelationId, RowExclusiveLock);
+ foreach(lc, vals)
+ {
+ Oid oid = GetNewOidWithIndex(pg_dict, DictOidIndexId,
+ Anum_pg_dict_oid);
+ char *entry = strVal(lfirst(lc));
+
+ /*
+ * Entries are stored in a name field, for easier syscache lookup, so
+ * check the length to make sure it's within range.
+ */
+ if (strlen(entry) > (NAMEDATALEN - 1))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_NAME),
+ errmsg("invalid dict entry \"%s\"", entry),
+ errdetail("Entries must be %d bytes or less.",
+ NAMEDATALEN - 1)));
+
+ namestrcpy(&dictentry, entry);
+
+ values[Anum_pg_dict_oid - 1] = ObjectIdGetDatum(oid);
+ values[Anum_pg_dict_dicttypid - 1] = ObjectIdGetDatum(dictTypeOid);
+ values[Anum_pg_dict_dictentry - 1] = NameGetDatum(&dictentry);
+
+ tup = heap_form_tuple(RelationGetDescr(pg_dict), values, nulls);
+
+ CatalogTupleInsert(pg_dict, tup);
+ heap_freetuple(tup);
+ }
+
+ /* clean up */
+ table_close(pg_dict, RowExclusiveLock);
+}
+
+/*
+ * Returns all the entries for the dictinary with given Oid. Entries are sorted
+ * by dictentry. Note that shorter entries are considered smaller, i.e. 'abc'
+ * goes before 'abcdef'. The memory is allocated in the caller's memory context.
+ *
+ * If there are no entries a valid but empty dictionary is returned.
+ */
+Dictionary
+DictEntriesRead(Oid dictTypeOid)
+{
+ Relation pg_dict;
+ ScanKeyData key[1];
+ SysScanDesc scan;
+ HeapTuple tup;
+ uint32 entries_allocated = 8;
+ Dictionary dict = (Dictionary)palloc(sizeof(DictionaryData));
+
+ dict->nentries = 0;
+ dict->entries = (DictEntry*)palloc(entries_allocated*sizeof(DictEntry));
+
+ pg_dict = table_open(DictRelationId, RowExclusiveLock);
+
+ ScanKeyInit(&key[0],
+ Anum_pg_dict_dicttypid,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(dictTypeOid));
+
+ scan = systable_beginscan(pg_dict, DictTypIdEntryIndexId, true,
+ NULL, 1, key);
+
+ while (HeapTupleIsValid(tup = systable_getnext(scan)))
+ {
+ if(dict->nentries == entries_allocated)
+ {
+ entries_allocated = entries_allocated * 2;
+ dict->entries = (DictEntry*)repalloc(dict->entries, entries_allocated*sizeof(DictEntry));
+ }
+
+ dict->entries[dict->nentries].oid = ((Form_pg_dict) GETSTRUCT(tup))->oid;
+ dict->entries[dict->nentries].dictentry = ((Form_pg_dict) GETSTRUCT(tup))->dictentry;
+ dict->nentries++;
+ }
+
+ systable_endscan(scan);
+ table_close(pg_dict, RowExclusiveLock);
+
+ return dict;
+}
+
+/*
+ * Deletes all the entries for the dictinary with given Oid.
+ */
+void
+DictEntriesDelete(Oid dictTypeOid)
+{
+ Relation pg_dict;
+ ScanKeyData key[1];
+ SysScanDesc scan;
+ HeapTuple tup;
+
+ pg_dict = table_open(DictRelationId, RowExclusiveLock);
+
+ ScanKeyInit(&key[0],
+ Anum_pg_dict_dicttypid,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(dictTypeOid));
+
+ scan = systable_beginscan(pg_dict, DictTypIdEntryIndexId, true,
+ NULL, 1, key);
+
+ while (HeapTupleIsValid(tup = systable_getnext(scan)))
+ {
+ CatalogTupleDelete(pg_dict, &tup->t_self);
+ }
+
+ systable_endscan(scan);
+ table_close(pg_dict, RowExclusiveLock);
+}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index 9b92b04242..5bbbbe09af 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -46,6 +46,7 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_constraint.h"
#include "catalog/pg_depend.h"
+#include "catalog/pg_dict.h"
#include "catalog/pg_enum.h"
#include "catalog/pg_language.h"
#include "catalog/pg_namespace.h"
@@ -679,6 +680,14 @@ RemoveTypeById(Oid typeOid)
if (((Form_pg_type) GETSTRUCT(tup))->typtype == TYPTYPE_RANGE)
RangeDelete(typeOid);
+ /*
+ * If it is a dictionary type, delete the pg_dict entries too; we don't
+ * bother with making a dependency entry for that, so it has to be done
+ * "by hand" here.
+ */
+ if (((Form_pg_type) GETSTRUCT(tup))->typtype == TYPTYPE_DICT)
+ DictEntriesDelete(typeOid);
+
ReleaseSysCache(tup);
table_close(relation, RowExclusiveLock);
@@ -763,8 +772,8 @@ DefineDomain(CreateDomainStmt *stmt)
/*
* Base type must be a plain base type, a composite type, another domain,
- * an enum or a range type. Domains over pseudotypes would create a
- * security hole. (It would be shorter to code this to just check for
+ * a dict, an enum or a range type. Domains over pseudotypes would create
+ * a security hole. (It would be shorter to code this to just check for
* pseudotypes; but it seems safer to call out the specific typtypes that
* are supported, rather than assume that all future typtypes would be
* automatically supported.)
@@ -772,6 +781,7 @@ DefineDomain(CreateDomainStmt *stmt)
typtype = baseType->typtype;
if (typtype != TYPTYPE_BASE &&
typtype != TYPTYPE_COMPOSITE &&
+ typtype != TYPTYPE_DICT &&
typtype != TYPTYPE_DOMAIN &&
typtype != TYPTYPE_ENUM &&
typtype != TYPTYPE_RANGE &&
@@ -1129,6 +1139,143 @@ DefineDomain(CreateDomainStmt *stmt)
}
+/*
+ * DefineDictionary
+ * Registers a new dictionary.
+ */
+ObjectAddress
+DefineDictionary(CreateDictionaryStmt * stmt)
+{
+ char *dictName;
+ char *dictArrayName;
+ Oid dictNamespace;
+ AclResult aclresult;
+ Oid old_type_oid;
+ Oid dictArrayOid;
+ ObjectAddress dictTypeAddr;
+
+ /* Convert list of names to a name and namespace */
+ dictNamespace = QualifiedNameGetCreationNamespace(stmt->typeName,
+ &dictName);
+
+ /* Check we have creation rights in target namespace */
+ aclresult = pg_namespace_aclcheck(dictNamespace, GetUserId(), ACL_CREATE);
+ if (aclresult != ACLCHECK_OK)
+ aclcheck_error(aclresult, OBJECT_SCHEMA,
+ get_namespace_name(dictNamespace));
+
+ /* AALEKSEEV TODO make sure this is allowed only for JSONB */
+
+ /*
+ * Check for collision with an existing type name. If there is one and
+ * it's an autogenerated array, we can rename it out of the way.
+ */
+ old_type_oid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid,
+ CStringGetDatum(dictName),
+ ObjectIdGetDatum(dictNamespace));
+ if (OidIsValid(old_type_oid))
+ {
+ if (!moveArrayTypeName(old_type_oid, dictName, dictNamespace))
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_OBJECT),
+ errmsg("type \"%s\" already exists", dictName)));
+ }
+
+ /* Allocate OID for array type */
+ dictArrayOid = AssignTypeArrayOid();
+
+ /* Create the pg_type entry */
+ dictTypeAddr =
+ TypeCreate(InvalidOid, /* no predetermined type OID */
+ dictName, /* type name */
+ dictNamespace, /* namespace */
+ InvalidOid, /* relation oid (n/a here) */
+ 0, /* relation kind (ditto) */
+ GetUserId(), /* owner's ID */
+ -1, /* internal size: varlena, as for JSONB */
+ TYPTYPE_DICT, /* type-type: dictionary */
+ TYPCATEGORY_USER, /* type-category: user, as for JSONB */
+ false, /* dict types are never preferred */
+ DEFAULT_TYPDELIM, /* array element delimiter */
+ F_DICTIONARY_IN, /* input procedure */
+ F_DICTIONARY_OUT, /* output procedure */
+ InvalidOid, /* receive procedure: none */
+ InvalidOid, /* send procedure: none */
+ InvalidOid, /* typmodin procedure - none */
+ InvalidOid, /* typmodout procedure - none */
+ InvalidOid, /* analyze procedure - default */
+ InvalidOid, /* subscript procedure - none */
+ InvalidOid, /* element type ID */
+ false, /* this is not an array type */
+ dictArrayOid, /* array type we are about to create */
+ InvalidOid, /* base type ID (only for domains) */
+ NULL, /* never a default type value */
+ NULL, /* binary default isn't sent either */
+ false, /* passed by value: same as for JSONB */
+ TYPALIGN_INT, /* int alignment */
+ TYPSTORAGE_EXTENDED, /* TOAST strategy: fully toastable, as
+ * JSONB */
+ -1, /* typMod (Domains only) */
+ 0, /* Array dimensions of typbasetype */
+ false, /* Type NOT NULL */
+ InvalidOid); /* type's collation */
+
+ /*
+ * Create the array type that goes with it.
+ */
+ dictArrayName = makeArrayTypeName(dictName, dictNamespace);
+
+ TypeCreate(dictArrayOid, /* force assignment of this type OID */
+ dictArrayName, /* type name */
+ dictNamespace, /* namespace */
+ InvalidOid, /* relation oid (n/a here) */
+ 0, /* relation kind (ditto) */
+ GetUserId(), /* owner's ID */
+ -1, /* internal size (always varlena) */
+ TYPTYPE_BASE, /* type-type (base type) */
+ TYPCATEGORY_ARRAY, /* type-category (array) */
+ false, /* array types are never preferred */
+ DEFAULT_TYPDELIM, /* array element delimiter */
+ F_ARRAY_IN, /* input procedure */
+ F_ARRAY_OUT, /* output procedure */
+ F_ARRAY_RECV, /* receive procedure */
+ F_ARRAY_SEND, /* send procedure */
+ InvalidOid, /* typmodin procedure - none */
+ InvalidOid, /* typmodout procedure - none */
+ F_ARRAY_TYPANALYZE, /* analyze procedure */
+ F_ARRAY_SUBSCRIPT_HANDLER, /* array subscript procedure */
+ dictTypeAddr.objectId, /* element type ID */
+ true, /* yes this is an array type */
+ InvalidOid, /* no further array type */
+ InvalidOid, /* base type ID */
+ NULL, /* never a default type value */
+ NULL, /* binary default isn't sent either */
+ false, /* passed by value: same as for JSONB */
+ TYPALIGN_INT, /* enums have int align, so do their arrays */
+ TYPSTORAGE_EXTENDED, /* ARRAY is always toastable */
+ -1, /* typMod (Domains only) */
+ 0, /* Array dimensions of typbasetype */
+ false, /* Type NOT NULL */
+ InvalidOid); /* type's collation */
+
+ pfree(dictArrayName);
+
+ /* Enter the dict's entries into pg_dict */
+ DictEntriesCreate(dictTypeAddr.objectId, stmt->vals);
+
+ /* Create casts to and from JSONB */
+ CastCreate(JSONBOID, dictTypeAddr.objectId, F_JSONB_DICTIONARY, 'a', 'f', DEPENDENCY_INTERNAL);
+ CastCreate(dictTypeAddr.objectId, JSONBOID, F_DICTIONARY_JSONB, 'i', 'f', DEPENDENCY_INTERNAL);
+
+ /*
+ * Create explicit cast to bytea. This is convenient for debugging purposes.
+ * Casting bytea to a dictionary type is dangerous and thus not supported.
+ */
+ CastCreate(dictTypeAddr.objectId, BYTEAOID, F_DICTIONARY_BYTEA, 'e', 'f', DEPENDENCY_INTERNAL);
+
+ return dictTypeAddr;
+}
+
/*
* DefineEnum
* Registers a new enum.
diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c
index f9460ae506..7ad3de3a00 100644
--- a/src/backend/executor/functions.c
+++ b/src/backend/executor/functions.c
@@ -1708,6 +1708,7 @@ check_sql_fn_retval(List *queryTreeLists,
if (fn_typtype == TYPTYPE_BASE ||
fn_typtype == TYPTYPE_DOMAIN ||
+ fn_typtype == TYPTYPE_DICT ||
fn_typtype == TYPTYPE_ENUM ||
fn_typtype == TYPTYPE_RANGE ||
fn_typtype == TYPTYPE_MULTIRANGE)
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 836f427ea8..04b9d573f4 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -4440,6 +4440,18 @@ _copyCompositeTypeStmt(const CompositeTypeStmt *from)
return newnode;
}
+static CreateDictionaryStmt *
+_copyCreateDictionaryStmt(const CreateDictionaryStmt *from)
+{
+ CreateDictionaryStmt *newnode = makeNode(CreateDictionaryStmt);
+
+ COPY_NODE_FIELD(typeName);
+ COPY_NODE_FIELD(baseTypeName);
+ COPY_NODE_FIELD(vals);
+
+ return newnode;
+}
+
static CreateEnumStmt *
_copyCreateEnumStmt(const CreateEnumStmt *from)
{
@@ -6181,6 +6193,9 @@ copyObjectImpl(const void *from)
case T_CompositeTypeStmt:
retval = _copyCompositeTypeStmt(from);
break;
+ case T_CreateDictionaryStmt:
+ retval = _copyCreateDictionaryStmt(from);
+ break;
case T_CreateEnumStmt:
retval = _copyCreateEnumStmt(from);
break;
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index e013c1bbfe..381d8b46f5 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -1962,6 +1962,16 @@ _equalCompositeTypeStmt(const CompositeTypeStmt *a, const CompositeTypeStmt *b)
return true;
}
+static bool
+_equalCreateDictionaryStmt(const CreateDictionaryStmt *a, const CreateDictionaryStmt *b)
+{
+ COMPARE_NODE_FIELD(typeName);
+ COMPARE_NODE_FIELD(baseTypeName);
+ COMPARE_NODE_FIELD(vals);
+
+ return true;
+}
+
static bool
_equalCreateEnumStmt(const CreateEnumStmt *a, const CreateEnumStmt *b)
{
@@ -3984,6 +3994,9 @@ equal(const void *a, const void *b)
case T_CompositeTypeStmt:
retval = _equalCompositeTypeStmt(a, b);
break;
+ case T_CreateDictionaryStmt:
+ retval = _equalCreateDictionaryStmt(a, b);
+ break;
case T_CreateEnumStmt:
retval = _equalCreateEnumStmt(a, b);
break;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c9941d9cb4..2ac632e025 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -5972,6 +5972,14 @@ DefineStmt:
n->coldeflist = $6;
$$ = (Node *)n;
}
+ | CREATE TYPE_P any_name AS DICTIONARY OF any_name '(' opt_enum_val_list ')'
+ {
+ CreateDictionaryStmt *n = makeNode(CreateDictionaryStmt);
+ n->typeName = $3;
+ n->baseTypeName = $7;
+ n->vals = $9;
+ $$ = (Node *)n;
+ }
| CREATE TYPE_P any_name AS ENUM_P '(' opt_enum_val_list ')'
{
CreateEnumStmt *n = makeNode(CreateEnumStmt);
diff --git a/src/backend/parser/parse_coerce.c b/src/backend/parser/parse_coerce.c
index c4e958e4aa..e489dd9421 100644
--- a/src/backend/parser/parse_coerce.c
+++ b/src/backend/parser/parse_coerce.c
@@ -279,7 +279,22 @@ coerce_type(ParseState *pstate, Node *node,
if (baseTypeId == INTERVALOID)
inputTypeMod = baseTypeMod;
else
- inputTypeMod = -1;
+ {
+ if((int32)targetTypeId == targetTypeMod)
+ {
+ /*
+ * For dictionaries internally we set typmod to a dictionary
+ * OID. This allows dictionary_in() and jsonb_dictionary() to
+ * distinguish one dictionary from another.
+ *
+ * It also allows us to recognize that the target type is
+ * a dictionary here and pass a correct typmod.
+ */
+ inputTypeMod = targetTypeMod;
+ }
+ else
+ inputTypeMod = -1;
+ }
baseType = typeidType(baseTypeId);
diff --git a/src/backend/parser/parse_type.c b/src/backend/parser/parse_type.c
index 307114a30d..868b24fd9f 100644
--- a/src/backend/parser/parse_type.c
+++ b/src/backend/parser/parse_type.c
@@ -259,6 +259,10 @@ LookupTypeNameOid(ParseState *pstate, const TypeName *typeName, bool missing_ok)
* This is equivalent to LookupTypeName, except that this will report
* a suitable error message if the type cannot be found or is not defined.
* Callers of this can therefore assume the result is a fully valid type.
+ *
+ * For a dictionary type it's OID is returned as a typmod. This is used by
+ * dictionary_in() and jsonb_dictionary() to distinguish one dictionary from
+ * another.
*/
Type
typenameType(ParseState *pstate, const TypeName *typeName, int32 *typmod_p)
@@ -305,6 +309,10 @@ typenameTypeId(ParseState *pstate, const TypeName *typeName)
*
* This is equivalent to typenameType, but we only hand back the type OID
* and typmod, not the syscache entry.
+ *
+ * For a dictionary type it's OID is returned as a typmod. This is used by
+ * dictionary_in() and jsonb_dictionary() to distinguish one dictionary from
+ * another.
*/
void
typenameTypeIdAndMod(ParseState *pstate, const TypeName *typeName,
@@ -327,6 +335,10 @@ typenameTypeIdAndMod(ParseState *pstate, const TypeName *typeName,
* looked up, and is passed as "typ".
*
* pstate is only used for error location info, and may be NULL.
+ *
+ * For a dictionary type it's OID is returned as a typmod. This is used by
+ * dictionary_in() and jsonb_dictionary() to distinguish one dictionary from
+ * another.
*/
static int32
typenameTypeMod(ParseState *pstate, const TypeName *typeName, Type typ)
@@ -339,6 +351,10 @@ typenameTypeMod(ParseState *pstate, const TypeName *typeName, Type typ)
ArrayType *arrtypmod;
ParseCallbackState pcbstate;
+ /* For a dictionary return it's OID. */
+ if (((Form_pg_type) GETSTRUCT(typ))->typtype == TYPTYPE_DICT)
+ return (int32)((Form_pg_type) GETSTRUCT(typ))->oid;
+
/* Return prespecified typmod if no typmod expressions */
if (typeName->typmods == NIL)
return typeName->typemod;
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 0e7b7b3138..cffef5d707 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -174,6 +174,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree)
case T_CreateCastStmt:
case T_CreateConversionStmt:
case T_CreateDomainStmt:
+ case T_CreateDictionaryStmt:
case T_CreateEnumStmt:
case T_CreateEventTrigStmt:
case T_CreateExtensionStmt:
@@ -1621,6 +1622,10 @@ ProcessUtilitySlow(ParseState *pstate,
}
break;
+ case T_CreateDictionaryStmt: /* CREATE TYPE AS DICTIONARY OF */
+ address = DefineDictionary((CreateDictionaryStmt *) parsetree);
+ break;
+
case T_CreateEnumStmt: /* CREATE TYPE AS ENUM */
address = DefineEnum((CreateEnumStmt *) parsetree);
break;
@@ -2766,6 +2771,10 @@ CreateCommandTag(Node *parsetree)
tag = CMDTAG_CREATE_TYPE;
break;
+ case T_CreateDictionaryStmt:
+ tag = CMDTAG_CREATE_TYPE;
+ break;
+
case T_CreateEnumStmt:
tag = CMDTAG_CREATE_TYPE;
break;
@@ -3413,6 +3422,10 @@ GetCommandLogLevel(Node *parsetree)
lev = LOGSTMT_DDL;
break;
+ case T_CreateDictionaryStmt:
+ lev = LOGSTMT_DDL;
+ break;
+
case T_CreateEnumStmt:
lev = LOGSTMT_DDL;
break;
diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile
index 7c722ea2ce..a7ec3d89f3 100644
--- a/src/backend/utils/adt/Makefile
+++ b/src/backend/utils/adt/Makefile
@@ -30,6 +30,7 @@ OBJS = \
datetime.o \
datum.o \
dbsize.o \
+ dictionaries.o \
domains.o \
encode.o \
enum.o \
diff --git a/src/backend/utils/adt/dictionaries.c b/src/backend/utils/adt/dictionaries.c
new file mode 100644
index 0000000000..7415ae806d
--- /dev/null
+++ b/src/backend/utils/adt/dictionaries.c
@@ -0,0 +1,476 @@
+/*-------------------------------------------------------------------------
+ *
+ * dictionaries.c
+ * Conversion functions for dictionary types.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/utils/adt/dictionaries.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "catalog/pg_dict.h"
+#include "utils/fmgrprotos.h"
+#include "utils/jsonb.h"
+
+/*
+ * When compressing a data we treat it as a BLOB, in other words we don't
+ * assume anything regarding its internal representation. This is not
+ * necessarily the best and/or the only possible approach. However, it is
+ * universal and can be reused for JSONB, TEXT, XML and other types. Alternative
+ * compression methods can be introduced in the future, so that the user will
+ * be able to choose the best one for the task.
+ *
+ * The compressed data is stored in the following format:
+ *
+ * (struct varlena)
+ * dictionary_id [uint32]
+ * decompressed_size [uint32]
+ * algorithm_version [uint8]
+ *
+ * (repeated) {
+ * number of bytes to copy as-is [uint8]
+ * ... bytes to copy as-is ...
+ * dictionary entry id, or 0 to skip [uint32]
+ * }
+ *
+ * Compressed data is a variable-length type and thus has the 'struct varlena'
+ * header. The code below doesn't consider it being a part of the payload
+ * (e.g. see the DICT_COMP_HEADER_SIZE definition).
+ *
+ * Storing dictionary_id may seem redundant, but without it dictionary_out()
+ * and dictionary_jsonb() have no way to distinguish one dictionary type from
+ * another. dictionary_in() and jsonb_dictionary() get the dictionary id through
+ * the 'typemod' argument.
+ *
+ * Currently algorithm_version is always 0. In the future it will allow us to
+ * introduce new features (e.g. usage of varints) and/or lazily migrate the
+ * data to other compression methods.
+ *
+ * Compression and decompression are implemented based on a binary search over
+ * DictEntry[] array (see pg_dict.h). For compression the array is sorted by
+ * dictentries and for decompression - by oid's.
+ *
+ */
+
+/* Size of the header in the compressed data */
+#define DICT_COMP_HEADER_SIZE (sizeof(uint32)*2 + sizeof(uint8))
+
+/* Extracts dictionary_id from the compressed data */
+#define DICT_COMP_DICTIONARY_ID(hdrp) \
+ (*(uint32*)hdrp)
+
+/* Extracts decompressed_size from the compressed data */
+#define DICT_COMP_DECOMPRESSED_SIZE(hdrp) \
+ (*(uint32*)((uint8*)hdrp + sizeof(uint32)))
+
+/* Extracts algorithm_version from the compressed data */
+#define DICT_COMP_ALGORITHM_VERSION(hdrp) \
+ (*(uint8*)((uint8*)hdrp + sizeof(uint32)*2))
+
+/* Current algorithm_version */
+#define DICT_COMP_CURRENT_ALGORITHM_VERSION 0
+
+/*
+ * bsearch_arg() callback for finding DictEntry by oid.
+ */
+static int
+find_by_oid_cb(const void *key, const void *current, void *arg)
+{
+ /* Note that oids are unsigned, so we should be careful here */
+ if (*(Oid *) key < ((DictEntry *) current)->oid)
+ return -1;
+ else if (*(Oid *) key == ((DictEntry *) current)->oid)
+ return 0; /* found! */
+ else
+ return 1;
+}
+
+/*
+ * qsort() callback for sorting DictEntry[] by oids.
+ */
+static int
+sort_by_oids_cb(const void *left, const void *right)
+{
+ /* Note that oids are unsigned, so we should be careful here */
+ if (((DictEntry *) left)->oid < ((DictEntry *) right)->oid)
+ return -1;
+
+ /* Oids are unique so this callback will never return 0 */
+ return 1;
+}
+
+/*
+ * Finds a DictEntry which dictentry field matches *data and returns its Oid.
+ * If there are several matching entries, the largest is returned. The length
+ * of the found entry is written to *found_length on success. On failure
+ * InvalidOid is returned and found_length is zeroed.
+ *
+ * The implementation is similar to bsearch_arg(). The procedure can't be used
+ * directly because we are looking not by the exact match. We could generalize
+ * this case but the signature of the functions becomes so complicated that it
+ * doesn't seem to worth the effort.
+ */
+static Oid
+compress_find_oid(Dictionary dict, const uint8 *data, Size data_size, Size *found_length)
+{
+ int res;
+ int32 left = 0;
+ int32 right = dict->nentries - 1;
+ Size best_length = 0;
+ Oid best_match = InvalidOid;
+
+ while (left <= right)
+ {
+ int32 current = (left + right) / 2;
+
+ /* AALEKSEEV TODO FIXME optimize */
+ Size nbytes = strlen(dict->entries[current].dictentry.data);
+
+ if (nbytes > data_size)
+ {
+ /* current can be less or greater depending on the prefix */
+ res = memcmp(dict->entries[current].dictentry.data, data, data_size);
+
+ /* if prefixes match, current is greater */
+ if (res == 0)
+ res = 1;
+ }
+ else
+ res = memcmp(dict->entries[current].dictentry.data, data, nbytes);
+
+ if (res == 0) /* match found */
+ {
+ best_length = nbytes;
+ best_match = dict->entries[current].oid;
+
+ if (nbytes == data_size)
+ break;
+
+ /* maybe there is a larger match */
+ left = current + 1;
+ }
+ else if (res < 0) /* current is less */
+ left = current + 1;
+ else /* current is greater */
+ right = current - 1;
+ }
+
+ *found_length = best_length;
+ return best_match;
+}
+
+/*
+ * Finds a DictEntry by Oid using a binary search. The dictionary should be
+ * sorted by oids before the call. Returns NULL if nothing was found.
+ */
+static DictEntry *
+decompress_find_dictentry(Dictionary dict, Oid oid)
+{
+ return (DictEntry *) bsearch_arg(&oid, dict->entries, dict->nentries, sizeof(DictEntry), find_by_oid_cb, NULL);
+}
+
+/*
+ * Estimates the worst-case compressed size of the data of given size.
+ * Worst-case scenario happens when the dictionary consists of single-character
+ * entries. In this case every byte will be encoded as 6 bytes:
+ * 0x00, (0 bytes to copy as-is), 4 bytes of the entry Oid
+ *
+ * This procedure doesn't account for the header size.
+ *
+ * AALEKSEEV TODO FIXME don't allow dictionary entries shorter than 4 bytes?
+ */
+static Size
+worst_case_compressed_size(Size insize)
+{
+ return insize * 6;
+}
+
+/*
+ * Compresses the data using the provided dictionary. The dictionary should
+ * be sorted by dictentries before the call. Output buffer should be at least
+ * worst_case_compressed_size(src_size) bytes in size.
+ */
+static void
+compress(Dictionary dict,
+ const void *src_data_, Size src_size,
+ void *encoded_data_, Size *pencoded_size)
+{
+ Size nbytes;
+ Size inoffset;
+ Size outskipoffset = 0;
+ Size outoffset = 1;
+ uint8 skipbytes = 0;
+ const uint8 *src_data = src_data_;
+ uint8 *encoded_data = ((uint8 *) encoded_data_);
+
+ for (inoffset = 0; inoffset < src_size;)
+ {
+ Oid code = compress_find_oid(dict, &(src_data[inoffset]),
+ src_size - inoffset, &nbytes);
+
+ if (code == InvalidOid)
+ {
+ skipbytes++;
+ encoded_data[outoffset] = src_data[inoffset];
+ outoffset++;
+ inoffset++;
+
+ if (skipbytes == 255)
+ {
+ encoded_data[outskipoffset] = skipbytes;
+ encoded_data[outoffset++] = 0; /* InvalidOid */
+ encoded_data[outoffset++] = 0;
+ encoded_data[outoffset++] = 0;
+ encoded_data[outoffset++] = 0;
+ outskipoffset = outoffset++;
+ skipbytes = 0;
+ }
+ }
+ else
+ {
+ encoded_data[outskipoffset] = skipbytes;
+ encoded_data[outoffset++] = (code >> 24) & 0xFF;
+ encoded_data[outoffset++] = (code >> 16) & 0xFF;
+ encoded_data[outoffset++] = (code >> 8) & 0xFF;
+ encoded_data[outoffset++] = code & 0xFF;
+ outskipoffset = outoffset++;
+ skipbytes = 0;
+ inoffset += nbytes;
+ }
+ }
+
+ /* Double check that we didn't write out of buffer */
+ Assert(outoffset < worst_case_compressed_size(src_size));
+
+ encoded_data[outskipoffset] = skipbytes;
+ *pencoded_size = outoffset;
+}
+
+/*
+ * Decompresses the data using the provided dictionary. The dictionary should
+ * be sorted by oids before the call.
+ */
+static void
+decompress(Dictionary dict,
+ const void *encoded_data_, Size encoded_size,
+ void *decoded_data_, Size decoded_size)
+{
+ Size inoffset = 0;
+ Size outoffset = 0;
+ Oid code;
+ uint8 skipbytes;
+ const uint8 *encoded_data = ((uint8 *) encoded_data_);
+ uint8 *decoded_data = decoded_data_;
+
+ for (inoffset = 0; inoffset < encoded_size;)
+ {
+ skipbytes = encoded_data[inoffset++];
+
+ if (skipbytes > decoded_size - outoffset)
+ {
+ /* AALEKSEEV TODO FIXME write a proper error message */
+ elog(ERROR, "skipbytes > decoded_size - outoffset");
+ }
+
+ if (skipbytes > encoded_size - inoffset)
+ {
+ /* AALEKSEEV TODO FIXME write a proper error message */
+ elog(ERROR, "skipbytes > encoded_size - inoffset");
+ }
+
+ memcpy(
+ &(decoded_data[outoffset]),
+ &(encoded_data[inoffset]),
+ skipbytes
+ );
+
+ outoffset += skipbytes;
+ inoffset += skipbytes;
+
+ if (encoded_size == inoffset && decoded_size == outoffset)
+ break; /* end of input - its OK */
+
+ if (encoded_size - inoffset < 4)
+ {
+ /* AALEKSEEV TODO FIXME write a proper error message */
+ elog(ERROR, "encoded_size - inoffset < 4");
+ }
+
+ code = (Oid) encoded_data[inoffset++];
+ code = (code << 8) | (Oid) encoded_data[inoffset++];
+ code = (code << 8) | (Oid) encoded_data[inoffset++];
+ code = (code << 8) | (Oid) encoded_data[inoffset++];
+
+ if (code != InvalidOid)
+ {
+ Size entrylen;
+ DictEntry *entry = decompress_find_dictentry(dict, code);
+
+ if (entry == NULL)
+ {
+ /* AALEKSEEV TODO FIXME write a proper error message */
+ elog(ERROR, "entry == NULL");
+ }
+
+ Assert(entry->oid == code);
+
+ /* AALEKSEEV TODO FIXME optimize */
+ entrylen = strlen(entry->dictentry.data);
+
+ if (entrylen > decoded_size - outoffset)
+ {
+ /* AALEKSEEV TODO FIXME write a proper error message */
+ elog(ERROR, "entrylen > decoded_size - outoffset");
+ }
+
+ memcpy(
+ &(decoded_data[outoffset]),
+ entry->dictentry.data,
+ entrylen
+ );
+
+ outoffset += entrylen;
+ }
+ }
+}
+
+/*
+ * Converts a cstring to a dictionary.
+ */
+Datum
+dictionary_in(PG_FUNCTION_ARGS)
+{
+ const char *instr = PG_GETARG_CSTRING(0);
+#ifdef NOT_USED
+ Oid typelem = PG_GETARG_OID(1);
+#endif
+ int32 typmod = PG_GETARG_INT32(2);
+
+ Assert(typmod != -1);
+
+ Jsonb *jsonb = DatumGetJsonbP(DirectFunctionCall1(jsonb_in, CStringGetDatum(instr)));
+ bytea *dict = DatumGetByteaP(DirectFunctionCall2(jsonb_dictionary, JsonbPGetDatum(jsonb), Int32GetDatum(typmod)));
+
+ PG_RETURN_BYTEA_P(dict);
+}
+
+/*
+ * Converts a dictionary to a cstring.
+ */
+Datum
+dictionary_out(PG_FUNCTION_ARGS)
+{
+ bytea *dict = PG_GETARG_BYTEA_P(0);
+ Jsonb *jsonb = DatumGetJsonbP(DirectFunctionCall1(dictionary_jsonb, PointerGetDatum(dict)));
+ const char *outstr = DatumGetCString(DirectFunctionCall1(jsonb_out, JsonbPGetDatum(jsonb)));
+
+ PG_RETURN_CSTRING(outstr);
+}
+
+/*
+ * Coverts JSONB to a dictionary type.
+ */
+Datum
+jsonb_dictionary(PG_FUNCTION_ARGS)
+{
+ Dictionary dict;
+ Jsonb *jsonb = PG_GETARG_JSONB_P(0);
+ int32 typmod = PG_GETARG_INT32(1);
+ uint8 *jsonb_data = (uint8 *) VARDATA(jsonb);
+ Size jsonb_data_size = VARSIZE(jsonb) - VARHDRSZ;
+ uint8 *encoded_buff,
+ *encoded_header,
+ *encoded_data;
+ Size encoded_size,
+ encoded_buff_size;
+
+ Assert(typmod != -1);
+
+ dict = DictEntriesRead(typmod);
+ qsort((void *) dict->entries, dict->nentries, sizeof(DictEntry), sort_by_oids_cb);
+
+ encoded_buff_size = VARHDRSZ + DICT_COMP_HEADER_SIZE + worst_case_compressed_size(jsonb_data_size);
+ encoded_buff = palloc(encoded_buff_size);
+ encoded_header = (uint8 *) VARDATA(encoded_buff);
+ encoded_data = encoded_header + DICT_COMP_HEADER_SIZE;
+
+ DICT_COMP_DICTIONARY_ID(encoded_header) = typmod;
+ DICT_COMP_DECOMPRESSED_SIZE(encoded_header) = jsonb_data_size;
+ DICT_COMP_ALGORITHM_VERSION(encoded_header) = DICT_COMP_CURRENT_ALGORITHM_VERSION;
+
+ encoded_size = encoded_buff_size - VARHDRSZ - DICT_COMP_HEADER_SIZE;
+
+ compress(dict, jsonb_data, jsonb_data_size,
+ encoded_data, &encoded_size);
+
+ encoded_size += VARHDRSZ + DICT_COMP_HEADER_SIZE;
+
+ encoded_buff = repalloc(encoded_buff, encoded_size);
+ SET_VARSIZE(encoded_buff, encoded_size);
+
+ pfree(dict);
+ PG_RETURN_BYTEA_P(encoded_buff);
+}
+
+/*
+ * Converts a dictionary type to JSONB.
+ */
+Datum
+dictionary_jsonb(PG_FUNCTION_ARGS)
+{
+ bytea *encoded_buff = PG_GETARG_BYTEA_P(0);
+ uint8 *encoded_header = (uint8 *) VARDATA(encoded_buff);
+ uint8 *encoded_data = encoded_header + DICT_COMP_HEADER_SIZE;
+ Size encoded_size = VARSIZE(encoded_buff) - VARHDRSZ - DICT_COMP_HEADER_SIZE;
+ int alg_version = DICT_COMP_ALGORITHM_VERSION(encoded_header);
+ Oid dictOid; /* cannot read until algorithm version is
+ * checked */
+ uint32 decoded_size; /* cannot read until algorithm version is
+ * checked */
+ Jsonb *jsonb;
+ uint8 *jsonb_data;
+ Dictionary dict;
+
+ if (alg_version > DICT_COMP_CURRENT_ALGORITHM_VERSION)
+ ereport(ERROR,
+ (
+ errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("Unsupported compression algorithm version"),
+ errdetail("Saved algorithm version is %d, current version is %d",
+ alg_version, DICT_COMP_CURRENT_ALGORITHM_VERSION),
+ errhint("The data is either corrupted or imported from "
+ "the future version of PostgreSQL")
+ ));
+
+ dictOid = DICT_COMP_DICTIONARY_ID(encoded_header);
+ decoded_size = DICT_COMP_DECOMPRESSED_SIZE(encoded_header);
+ dict = DictEntriesRead(dictOid);
+
+ jsonb = palloc(decoded_size + VARHDRSZ);
+ jsonb_data = (uint8 *) VARDATA(jsonb);
+
+ decompress(dict, encoded_data, encoded_size, jsonb_data, decoded_size);
+
+ decoded_size += VARHDRSZ;
+ SET_VARSIZE(jsonb, decoded_size);
+
+ pfree(dict);
+ PG_RETURN_JSONB_P(jsonb);
+}
+
+/*
+ * Converts a dictionary type to bytea.
+ */
+Datum
+dictionary_bytea(PG_FUNCTION_ARGS)
+{
+ bytea *compressed_data = PG_GETARG_BYTEA_P(0);
+ PG_RETURN_BYTEA_P(compressed_data);
+}
\ No newline at end of file
diff --git a/src/backend/utils/fmgr/funcapi.c b/src/backend/utils/fmgr/funcapi.c
index 9197b0f1e2..4f9dc52428 100644
--- a/src/backend/utils/fmgr/funcapi.c
+++ b/src/backend/utils/fmgr/funcapi.c
@@ -1292,6 +1292,7 @@ get_type_func_class(Oid typid, Oid *base_typeid)
case TYPTYPE_COMPOSITE:
return TYPEFUNC_COMPOSITE;
case TYPTYPE_BASE:
+ case TYPTYPE_DICT:
case TYPTYPE_ENUM:
case TYPTYPE_RANGE:
case TYPTYPE_MULTIRANGE:
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 786d592e2b..493ef56c30 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -10177,6 +10177,8 @@ dumpType(Archive *fout, const TypeInfo *tyinfo)
dumpDomain(fout, tyinfo);
else if (tyinfo->typtype == TYPTYPE_COMPOSITE)
dumpCompositeType(fout, tyinfo);
+ else if (tyinfo->typtype == TYPTYPE_DICT)
+ pg_log_error("AALEKSEEV TODO FIXME not implemented");
else if (tyinfo->typtype == TYPTYPE_ENUM)
dumpEnumType(fout, tyinfo);
else if (tyinfo->typtype == TYPTYPE_RANGE)
diff --git a/src/include/catalog/pg_dict.h b/src/include/catalog/pg_dict.h
new file mode 100644
index 0000000000..df9cff5437
--- /dev/null
+++ b/src/include/catalog/pg_dict.h
@@ -0,0 +1,76 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_dict.h
+ * definition of the "dict" system catalog (pg_dict)
+ *
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/catalog/pg_dict.h
+ *
+ * NOTES
+ * The Catalog.pm module reads this file and derives schema
+ * information.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_DICT_H
+#define PG_DICT_H
+
+#include "catalog/genbki.h"
+#include "catalog/pg_dict_d.h"
+
+#include "nodes/pg_list.h"
+
+/* ----------------
+ * pg_dict definition. cpp turns this into
+ * typedef struct FormData_pg_dict
+ * ----------------
+ */
+CATALOG(pg_dict,9861,DictRelationId)
+{
+ Oid oid; /* oid */
+ Oid dicttypid BKI_LOOKUP(pg_type); /* OID of owning dict type */
+ NameData dictentry; /* text representation of the dictionary entry */
+} FormData_pg_dict;
+
+/* ----------------
+ * Form_pg_dict corresponds to a pointer to a tuple with
+ * the format of pg_dict relation.
+ * ----------------
+ */
+typedef FormData_pg_dict *Form_pg_dict;
+
+DECLARE_UNIQUE_INDEX_PKEY(pg_dict_oid_index, 9862, DictOidIndexId, on pg_dict using btree(oid oid_ops));
+DECLARE_UNIQUE_INDEX(pg_dict_typid_entry_index, 9863, DictTypIdEntryIndexId, on pg_dict using btree(dicttypid oid_ops, dictentry name_ops));
+
+
+/*
+ * DictEntry type represents one entry in the given dictionary.
+ */
+typedef struct
+{
+ Oid oid;
+ NameData dictentry;
+} DictEntry;
+
+/*
+ * Dictionary type represents all the entries in the given dictionary.
+ */
+typedef struct
+{
+ uint32 nentries;
+ DictEntry *entries;
+} DictionaryData;
+
+typedef DictionaryData *Dictionary;
+
+/*
+ * prototypes for functions in pg_dict.c
+ */
+extern void DictEntriesCreate(Oid dictTypeOid, List *vals);
+extern Dictionary DictEntriesRead(Oid dictTypeOid);
+extern void DictEntriesDelete(Oid dictTypeOid);
+
+#endif /* PG_DICT_H */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6d378ff785..c6ec8737df 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10022,6 +10022,26 @@
proname => 'jsonb_path_match_opr', prorettype => 'bool',
proargtypes => 'jsonb jsonpath', prosrc => 'jsonb_path_match_opr' },
+# dictionaries
+{ oid => '9864', descr => 'Converts a cstring to a dictionary',
+ proname => 'dictionary_in', prorettype => 'any', proargtypes => 'cstring oid int4',
+ prosrc => 'dictionary_in' },
+{ oid => '9865', descr => 'Converts a dictionary to a cstring',
+ proname => 'dictionary_out', prorettype => 'cstring', proargtypes => 'any',
+ prosrc => 'dictionary_out' },
+{ oid => '9866', descr => 'Coverts JSONB to a dictionary type',
+ proname => 'jsonb_dictionary', proisstrict => 'f', provolatile => 's',
+ prorettype => 'any', proargtypes => 'jsonb int4',
+ prosrc => 'jsonb_dictionary' },
+{ oid => '9867', descr => 'Converts a dictionary type to JSONB',
+ proname => 'dictionary_jsonb', proisstrict => 'f', provolatile => 's',
+ prorettype => 'jsonb', proargtypes => 'any',
+ prosrc => 'dictionary_jsonb' },
+{ oid => '9868', descr => 'Converts a dictionary type to a byte array',
+ proname => 'dictionary_bytea', proisstrict => 'f', provolatile => 's',
+ prorettype => 'jsonb', proargtypes => 'any',
+ prosrc => 'dictionary_bytea' },
+
# historical int8/txid_snapshot variants of xid8 functions
{ oid => '2939', descr => 'I/O',
proname => 'txid_snapshot_in', prorettype => 'txid_snapshot',
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index 48a2559137..91c6bfae99 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -272,6 +272,7 @@ DECLARE_UNIQUE_INDEX(pg_type_typname_nsp_index, 2704, TypeNameNspIndexId, on pg_
*/
#define TYPTYPE_BASE 'b' /* base type (ordinary scalar type) */
#define TYPTYPE_COMPOSITE 'c' /* composite (e.g., table's rowtype) */
+#define TYPTYPE_DICT 'D' /* dictionary */
#define TYPTYPE_DOMAIN 'd' /* domain over another type */
#define TYPTYPE_ENUM 'e' /* enumerated type */
#define TYPTYPE_MULTIRANGE 'm' /* multirange type */
diff --git a/src/include/commands/typecmds.h b/src/include/commands/typecmds.h
index a17bedb851..5cc8e5ff99 100644
--- a/src/include/commands/typecmds.h
+++ b/src/include/commands/typecmds.h
@@ -24,6 +24,7 @@
extern ObjectAddress DefineType(ParseState *pstate, List *names, List *parameters);
extern void RemoveTypeById(Oid typeOid);
extern ObjectAddress DefineDomain(CreateDomainStmt *stmt);
+extern ObjectAddress DefineDictionary(CreateDictionaryStmt *stmt);
extern ObjectAddress DefineEnum(CreateEnumStmt *stmt);
extern ObjectAddress DefineRange(ParseState *pstate, CreateRangeStmt *stmt);
extern ObjectAddress AlterEnum(AlterEnumStmt *stmt);
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 340d28f4e1..ad1e3ad9f5 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -408,6 +408,7 @@ typedef enum NodeTag
T_DropOwnedStmt,
T_ReassignOwnedStmt,
T_CompositeTypeStmt,
+ T_CreateDictionaryStmt,
T_CreateEnumStmt,
T_CreateRangeStmt,
T_AlterEnumStmt,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index da02658c81..5e3aacd8a7 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -3567,6 +3567,18 @@ typedef struct CompositeTypeStmt
List *coldeflist; /* list of ColumnDef nodes */
} CompositeTypeStmt;
+/* ----------------------
+ * Create Type Statement, dictionary types
+ * ----------------------
+ */
+typedef struct CreateDictionaryStmt
+{
+ NodeTag type;
+ List *typeName; /* qualified name (list of String) */
+ List *baseTypeName; /* qualified base type name (list of String) */
+ List *vals; /* dictionary entries (list of String) */
+} CreateDictionaryStmt;
+
/* ----------------------
* Create Type Statement, enum types
* ----------------------
diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c
index b791c23f06..897051c1f7 100644
--- a/src/pl/plpgsql/src/pl_comp.c
+++ b/src/pl/plpgsql/src/pl_comp.c
@@ -2129,6 +2129,7 @@ build_datatype(HeapTuple typeTup, int32 typmod,
switch (typeStruct->typtype)
{
case TYPTYPE_BASE:
+ case TYPTYPE_DICT:
case TYPTYPE_ENUM:
case TYPTYPE_RANGE:
case TYPTYPE_MULTIRANGE:
diff --git a/src/test/modules/test_oat_hooks/test_oat_hooks.c b/src/test/modules/test_oat_hooks/test_oat_hooks.c
index 551da5d498..1cab22183a 100644
--- a/src/test/modules/test_oat_hooks/test_oat_hooks.c
+++ b/src/test/modules/test_oat_hooks/test_oat_hooks.c
@@ -772,6 +772,7 @@ nodetag_to_string(NodeTag tag)
case T_DropOwnedStmt: return "DropOwnedStmt"; break;
case T_ReassignOwnedStmt: return "ReassignOwnedStmt"; break;
case T_CompositeTypeStmt: return "CompositeTypeStmt"; break;
+ case T_CreateDictionaryStmt: return "CreateDictionaryStmt"; break;
case T_CreateEnumStmt: return "CreateEnumStmt"; break;
case T_CreateRangeStmt: return "CreateRangeStmt"; break;
case T_AlterEnumStmt: return "AlterEnumStmt"; break;
diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out
index 215eb899be..91bc491a6c 100644
--- a/src/test/regress/expected/oidjoins.out
+++ b/src/test/regress/expected/oidjoins.out
@@ -182,6 +182,7 @@ NOTICE: checking pg_description {classoid} => pg_class {oid}
NOTICE: checking pg_cast {castsource} => pg_type {oid}
NOTICE: checking pg_cast {casttarget} => pg_type {oid}
NOTICE: checking pg_cast {castfunc} => pg_proc {oid}
+NOTICE: checking pg_dict {dicttypid} => pg_type {oid}
NOTICE: checking pg_enum {enumtypid} => pg_type {oid}
NOTICE: checking pg_namespace {nspowner} => pg_authid {oid}
NOTICE: checking pg_conversion {connamespace} => pg_namespace {oid}
diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out
index 86d755aa44..5dc4d0ab01 100644
--- a/src/test/regress/expected/opr_sanity.out
+++ b/src/test/regress/expected/opr_sanity.out
@@ -388,11 +388,12 @@ WHERE 'cstring'::regtype = ANY (p1.proargtypes)
AND NOT EXISTS(SELECT 1 FROM pg_conversion WHERE conproc = p1.oid)
AND p1.oid != 'shell_in(cstring)'::regprocedure
ORDER BY 1;
- oid | proname
-------+--------------
+ oid | proname
+------+---------------
2293 | cstring_out
2501 | cstring_send
-(2 rows)
+ 9864 | dictionary_in
+(3 rows)
-- Likewise, look for functions that return cstring and aren't datatype output
-- functions nor typmod output functions.
@@ -405,11 +406,12 @@ WHERE p1.prorettype = 'cstring'::regtype
AND NOT EXISTS(SELECT 1 FROM pg_type WHERE typmodout = p1.oid)
AND p1.oid != 'shell_out(void)'::regprocedure
ORDER BY 1;
- oid | proname
-------+--------------
+ oid | proname
+------+----------------
2292 | cstring_in
2500 | cstring_recv
-(2 rows)
+ 9865 | dictionary_out
+(3 rows)
-- Check for length inconsistencies between the various argument-info arrays.
SELECT p1.oid, p1.proname
--
2.35.1
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: [PATCH] Compression dictionaries for JSONB
@ 2022-06-01 21:29 Jacob Champion <[email protected]>
parent: Aleksander Alekseev <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Jacob Champion @ 2022-06-01 21:29 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
On Wed, Jun 1, 2022 at 1:44 PM Aleksander Alekseev
<[email protected]> wrote:
> This is a follow-up thread to `RFC: compression dictionaries for JSONB` [1]. I would like to share my current progress in order to get early feedback. The patch is currently in a draft state but implements the basic functionality. I did my best to account for all the great feedback I previously got from Alvaro and Matthias.
I'm coming up to speed with this set of threads -- the following is
not a complete review by any means, and please let me know if I've
missed some of the history.
> SELECT * FROM pg_dict;
> oid | dicttypid | dictentry
> -------+-----------+-----------
> 39476 | 39475 | aaa
> 39477 | 39475 | bbb
> (2 rows)
I saw there was some previous discussion about dictionary size. It
looks like this approach would put all dictionaries into a shared OID
pool. Since I don't know what a "standard" use case is, is there any
risk of OID exhaustion for larger deployments with many dictionaries?
Or is 2**32 so comparatively large that it's not really a serious
concern?
> When `mydict` sees 'aaa' in the document, it replaces it with the corresponding code, in this case - 39476. For more details regarding the compression algorithm and choosen compromises please see the comments in the patch.
I see the algorithm description, but I'm curious to know whether it's
based on some other existing compression scheme, for the sake of
comparison. It seems like it shares similarities with the Snappy
scheme?
Could you talk more about what the expected ratios and runtime
characteristics are? Best I can see is that compression runtime is
something like O(n * e * log d) where n is the length of the input, e
is the maximum length of a dictionary entry, and d is the number of
entries in the dictionary. Since e and d are constant for a given
static dictionary, how well the dictionary is constructed is
presumably important.
> In pg_type `mydict` has typtype = TYPTYPE_DICT. It works the same way as TYPTYPE_BASE with only difference: corresponding `<type>_in` (pg_type.typinput) and `<another-type>_<type>` (pg_cast.castfunc) procedures receive the dictionary Oid as a `typmod` argument. This way the procedures can distinguish `mydict1` from `mydict2` and use the proper compression dictionary.
>
> The approach with alternative `typmod` role is arguably a bit hacky, but it was the less invasive way to implement the feature I've found. I'm open to alternative suggestions.
Haven't looked at this closely enough to develop an opinion yet.
> Current limitations (todo):
> - ALTER TYPE is not implemented
That reminds me. How do people expect to generate a "good" dictionary
in practice? Would they somehow get the JSONB representations out of
Postgres and run a training program over the blobs? I see some
reference to training functions in the prior threads but don't see any
breadcrumbs in the code.
> - Alternative compression algorithms. Note that this will not require any further changes in the catalog, only the values we write to pg_type and pg_cast will differ.
Could you expand on this? I.e. why would alternative algorithms not
need catalog changes? It seems like the only schemes that could be
used with pg_catalog.pg_dict are those that expect to map a byte
string to a number. Is that general enough to cover other standard
compression algorithms?
> Open questions:
> - Dictionary entries are currently stored as NameData, the same type that is used for enums. Are we OK with the accompanying limitations? Any alternative suggestions?
It does feel a little weird to have a hard limit on the entry length,
since that also limits the compression ratio. But it also limits the
compression runtime, so maybe it's a worthwhile tradeoff.
It also seems strange to use a dictionary of C strings to compress
binary data; wouldn't we want to be able to compress zero bytes too?
Hope this helps,
--Jacob
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: [PATCH] Compression dictionaries for JSONB
@ 2022-06-02 13:30 Aleksander Alekseev <[email protected]>
parent: Jacob Champion <[email protected]>
0 siblings, 2 replies; 7+ messages in thread
From: Aleksander Alekseev @ 2022-06-02 13:30 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
Hi Jacob,
Many thanks for your feedback!
> I saw there was some previous discussion about dictionary size. It
> looks like this approach would put all dictionaries into a shared OID
> pool. Since I don't know what a "standard" use case is, is there any
> risk of OID exhaustion for larger deployments with many dictionaries?
> Or is 2**32 so comparatively large that it's not really a serious
> concern?
I agree, this is a drawback of the current implementation. To be honest,
I simply followed the example of how ENUMs are implemented. I'm not 100% sure
if we should be worried here (apparently, freed OIDs are reused). I'm OK with
using a separate sequence if someone could second this. This is the first time
I'm altering the catalog so I'm not certain what the best practices are.
> I see the algorithm description, but I'm curious to know whether it's
> based on some other existing compression scheme, for the sake of
> comparison. It seems like it shares similarities with the Snappy
> scheme?
>
> Could you talk more about what the expected ratios and runtime
> characteristics are? Best I can see is that compression runtime is
> something like O(n * e * log d) where n is the length of the input, e
> is the maximum length of a dictionary entry, and d is the number of
> entries in the dictionary. Since e and d are constant for a given
> static dictionary, how well the dictionary is constructed is
> presumably important.
The algorithm is almost identical to the one I used in ZSON extension [1]
except the fact that ZSON uses 16-bit codes. In docs/benchmark.md you will find
approximate ratios to expect, etc. The reasons why this particular algorithm
was chosen are:
1. It was extensively tested in the past and seem to work OK for existing
ZSON users.
2. It doesn't use any knowledge regarding the data structure and thus can be
reused for TEXT/XML/etc as-is.
3. Previously we agreed that at some point users will be able to change the
algorithm (the same way as they can do it for TOAST now) so which algorithm
will be used in the first implementation is not that important. I simply
choose the already existing one.
> > Current limitations (todo):
> > - ALTER TYPE is not implemented
>
> That reminds me. How do people expect to generate a "good" dictionary
> in practice? Would they somehow get the JSONB representations out of
> Postgres and run a training program over the blobs? I see some
> reference to training functions in the prior threads but don't see any
> breadcrumbs in the code.
So far we agreed that in the first implementation it will be done manually.
In the future it will be possible to update the dictionaries automatically
during VACUUM. The idea of something similar to zson_learn() procedure, as
I recall, didn't get much support, so we probably will not have it, or at least
it is not a priority.
> > - Alternative compression algorithms. Note that this will not require any
> > further changes in the catalog, only the values we write to pg_type and
> > pg_cast will differ.
>
> Could you expand on this? I.e. why would alternative algorithms not
> need catalog changes? It seems like the only schemes that could be
> used with pg_catalog.pg_dict are those that expect to map a byte
> string to a number. Is that general enough to cover other standard
> compression algorithms?
Sure. When creating a new dictionary pg_type and pg_cast are modified like this:
=# CREATE TYPE mydict AS DICTIONARY OF JSONB ('abcdef', 'ghijkl');
CREATE TYPE
=# SELECT * FROM pg_type WHERE typname = 'mydict';
-[ RECORD 1 ]--+---------------
oid | 16397
typname | mydict
typnamespace | 2200
...
typarray | 16396
typinput | dictionary_in
typoutput | dictionary_out
...
=# SELECT c.*, p.proname FROM pg_cast AS c
LEFT JOIN pg_proc AS p
ON p.oid = c.castfunc
WHERE c.castsource = 16397 or c.casttarget = 16397;
-[ RECORD 1 ]-----------------
oid | 16400
castsource | 3802
casttarget | 16397
castfunc | 9866
castcontext | a
castmethod | f
proname | jsonb_dictionary
-[ RECORD 2 ]-----------------
oid | 16401
castsource | 16397
casttarget | 3802
castfunc | 9867
castcontext | i
castmethod | f
proname | dictionary_jsonb
-[ RECORD 3 ]-----------------
oid | 16402
castsource | 16397
casttarget | 17
castfunc | 9868
castcontext | e
castmethod | f
proname | dictionary_bytea
In order to add a new algorithm you simply need to provide alternatives
to dictionary_in / dictionary_out / jsonb_dictionary / dictionary_jsonb and
specify them in the catalog instead. The catalog schema will remain the same.
> It also seems strange to use a dictionary of C strings to compress
> binary data; wouldn't we want to be able to compress zero bytes too?
That's a good point. Again, here I simply followed the example of the ENUMs
implementation. Since compression dictionaries are intended to be used with
text-like types such as JSONB, (and also JSON, TEXT and XML in the future),
choosing Name type seemed to be a reasonable compromise. Dictionary entries are
most likely going to store JSON keys, common words used in the TEXT, etc.
However, I'm fine with any alternative scheme if somebody experienced with the
PostgreSQL catalog could second this.
[1]: https://github.com/afiskon/zson
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: [PATCH] Compression dictionaries for JSONB
@ 2022-06-02 22:21 Jacob Champion <[email protected]>
parent: Aleksander Alekseev <[email protected]>
1 sibling, 0 replies; 7+ messages in thread
From: Jacob Champion @ 2022-06-02 22:21 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
On Thu, Jun 2, 2022 at 6:30 AM Aleksander Alekseev
<[email protected]> wrote:
> > I saw there was some previous discussion about dictionary size. It
> > looks like this approach would put all dictionaries into a shared OID
> > pool. Since I don't know what a "standard" use case is, is there any
> > risk of OID exhaustion for larger deployments with many dictionaries?
> > Or is 2**32 so comparatively large that it's not really a serious
> > concern?
>
> I agree, this is a drawback of the current implementation. To be honest,
> I simply followed the example of how ENUMs are implemented. I'm not 100% sure
> if we should be worried here (apparently, freed OIDs are reused). I'm OK with
> using a separate sequence if someone could second this. This is the first time
> I'm altering the catalog so I'm not certain what the best practices are.
I think reuse should be fine (if a bit slower, but offhand that
doesn't seem like an important bottleneck). Users may be unamused to
find that one large dictionary has prevented the creation of any new
entries in other dictionaries, though. But again, I have no intuition
for the size of a production-grade compression dictionary, and maybe
it's silly to assume that normal use would ever reach the OID limit.
> > I see the algorithm description, but I'm curious to know whether it's
> > based on some other existing compression scheme, for the sake of
> > comparison. It seems like it shares similarities with the Snappy
> > scheme?
> >
> > Could you talk more about what the expected ratios and runtime
> > characteristics are? Best I can see is that compression runtime is
> > something like O(n * e * log d) where n is the length of the input, e
> > is the maximum length of a dictionary entry, and d is the number of
> > entries in the dictionary. Since e and d are constant for a given
> > static dictionary, how well the dictionary is constructed is
> > presumably important.
>
> The algorithm is almost identical to the one I used in ZSON extension [1]
> except the fact that ZSON uses 16-bit codes. In docs/benchmark.md you will find
> approximate ratios to expect, etc.
That's assuming a machine-trained dictionary, though, which isn't part
of the proposal now. Is there a performance/ratio sample for a "best
practice" hand-written dictionary?
> > That reminds me. How do people expect to generate a "good" dictionary
> > in practice? Would they somehow get the JSONB representations out of
> > Postgres and run a training program over the blobs? I see some
> > reference to training functions in the prior threads but don't see any
> > breadcrumbs in the code.
>
> So far we agreed that in the first implementation it will be done manually.
> In the future it will be possible to update the dictionaries automatically
> during VACUUM. The idea of something similar to zson_learn() procedure, as
> I recall, didn't get much support, so we probably will not have it, or at least
> it is not a priority.
Hm... I'm skeptical that a manually-constructed set of compression
dictionaries would be maintainable over time or at scale. But I'm not
the target audience so I will let others weigh in here instead.
> > > - Alternative compression algorithms. Note that this will not require any
> > > further changes in the catalog, only the values we write to pg_type and
> > > pg_cast will differ.
> >
> > Could you expand on this? I.e. why would alternative algorithms not
> > need catalog changes? It seems like the only schemes that could be
> > used with pg_catalog.pg_dict are those that expect to map a byte
> > string to a number. Is that general enough to cover other standard
> > compression algorithms?
>
> Sure. When creating a new dictionary pg_type and pg_cast are modified like this:
>
> =# CREATE TYPE mydict AS DICTIONARY OF JSONB ('abcdef', 'ghijkl');
> CREATE TYPE
> =# SELECT * FROM pg_type WHERE typname = 'mydict';
> -[ RECORD 1 ]--+---------------
> oid | 16397
> typname | mydict
> typnamespace | 2200
> ...
> typarray | 16396
> typinput | dictionary_in
> typoutput | dictionary_out
> ...
>
> =# SELECT c.*, p.proname FROM pg_cast AS c
> LEFT JOIN pg_proc AS p
> ON p.oid = c.castfunc
> WHERE c.castsource = 16397 or c.casttarget = 16397;
> -[ RECORD 1 ]-----------------
> oid | 16400
> castsource | 3802
> casttarget | 16397
> castfunc | 9866
> castcontext | a
> castmethod | f
> proname | jsonb_dictionary
> -[ RECORD 2 ]-----------------
> oid | 16401
> castsource | 16397
> casttarget | 3802
> castfunc | 9867
> castcontext | i
> castmethod | f
> proname | dictionary_jsonb
> -[ RECORD 3 ]-----------------
> oid | 16402
> castsource | 16397
> casttarget | 17
> castfunc | 9868
> castcontext | e
> castmethod | f
> proname | dictionary_bytea
>
> In order to add a new algorithm you simply need to provide alternatives
> to dictionary_in / dictionary_out / jsonb_dictionary / dictionary_jsonb and
> specify them in the catalog instead. The catalog schema will remain the same.
The catalog schemas for pg_type and pg_cast would. But would the
current pg_dict schema be generally applicable to other cross-table
compression schemes? It seems narrowly tailored -- which is not a
problem for a proof of concept patch; I'm just not seeing how other
standard compression schemes might make use of an OID-to-NameData map.
My naive understanding is that they have their own dictionary
structures.
(You could of course hack in any general structure you needed by
treating pg_dict like a list of chunks, but that seems wasteful and
slow, especially given the 63-byte chunk limit, and even more likely
to exhaust the shared OID pool. I think LZMA dictionaries can be huge,
as one example.)
> > It also seems strange to use a dictionary of C strings to compress
> > binary data; wouldn't we want to be able to compress zero bytes too?
>
> That's a good point. Again, here I simply followed the example of the ENUMs
> implementation. Since compression dictionaries are intended to be used with
> text-like types such as JSONB, (and also JSON, TEXT and XML in the future),
> choosing Name type seemed to be a reasonable compromise. Dictionary entries are
> most likely going to store JSON keys, common words used in the TEXT, etc.
> However, I'm fine with any alternative scheme if somebody experienced with the
> PostgreSQL catalog could second this.
I think Matthias back in the first thread was hoping for the ability
to compress duplicated JSON objects as well; it seems like that
wouldn't be possible with the current scheme. (Again I have no
intuition for which use cases are must-haves.) I'm wondering if
pg_largeobject would be an alternative catalog to draw inspiration
from... specifically the use of bytea as the stored value, and of a
two-column primary key.
But take all my suggestions with a dash of salt :D I'm new to this space.
Thanks!
--Jacob
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: [PATCH] Compression dictionaries for JSONB
@ 2022-06-23 15:48 Simon Riggs <[email protected]>
parent: Aleksander Alekseev <[email protected]>
1 sibling, 1 reply; 7+ messages in thread
From: Simon Riggs @ 2022-06-23 15:48 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: Jacob Champion <[email protected]>; PostgreSQL Hackers <[email protected]>; Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
On Thu, 2 Jun 2022 at 14:30, Aleksander Alekseev
<[email protected]> wrote:
> > I saw there was some previous discussion about dictionary size. It
> > looks like this approach would put all dictionaries into a shared OID
> > pool. Since I don't know what a "standard" use case is, is there any
> > risk of OID exhaustion for larger deployments with many dictionaries?
> > Or is 2**32 so comparatively large that it's not really a serious
> > concern?
>
> I agree, this is a drawback of the current implementation. To be honest,
> I simply followed the example of how ENUMs are implemented. I'm not 100% sure
> if we should be worried here (apparently, freed OIDs are reused). I'm OK with
> using a separate sequence if someone could second this. This is the first time
> I'm altering the catalog so I'm not certain what the best practices are.
The goal of this patch is great, thank you for working on this (and ZSON).
The approach chosen has a few downsides that I'm not happy with yet.
* Assigning OIDs for each dictionary entry is not a great idea. I
don't see why you would need to do that; just assign monotonically
ascending keys for each dictionary, as we do for AttrNums.
* There is a limit on SQL statement size, which will effectively limit
the size of dictionaries, but the examples are unrealistically small,
so this isn't clear as a limitation, but it would be in practice. It
would be better to specify a filename, which can be read in when the
DDL executes. This can be put into pg_dump output in a similar way to
the COPY data for a table is, so once read in it stays static.
* The dictionaries are only allowed for certain datatypes. This should
not be specifically limited by this patch, i.e. user defined types
should not be rejected.
* Dictionaries have no versioning. Any list of data items changes over
time, so how do we express that? Enums were also invented as static
lists originally, then had to be modified later to accomodate
additions and revisions, so let's think about that now, even if we
don't add all of the commands in one go. Currently we would have to
create a whole new dictionary if even one word changes. Ideally, we
want the dictionary to have a top-level name and then have multiple
versions over time. Let's agree how we are going do these things, so
we can make sure the design and code allows for those future
enhancements.
i.e. how will we do ALTER TABLE ... UPGRADE DICTIONARY without causing
a table rewrite?
* Does the order of entries in the dictionary allow us to express a
priority? i.e. to allow Huffman coding.
Thanks for your efforts - this is a very important patch.
--
Simon Riggs http://www.EnterpriseDB.com/
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: [PATCH] Compression dictionaries for JSONB
@ 2022-06-28 12:37 Aleksander Alekseev <[email protected]>
parent: Simon Riggs <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Aleksander Alekseev @ 2022-06-28 12:37 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>; +Cc: Jacob Champion <[email protected]>; Simon Riggs <[email protected]>; Alvaro Herrera <[email protected]>; Matthias van de Meent <[email protected]>
Hi Simon,
Many thanks for your feedback!
I'm going to submit an updated version of the patch in a bit. I just
wanted to reply to some of your questions / comments.
> Dictionaries have no versioning. [...]
> Does the order of entries in the dictionary allow us to express a priority? i.e. to allow Huffman coding. [...]
This is something we discussed in the RFC thread. I got an impression
that the consensus was reached:
1. To simply use 32-bit codes in the compressed documents, instead of
16-bit ones as it was done in ZSON;
2. Not to use any sort of variable-length coding;
3. Not to use dictionary versions. New codes can be added to the
existing dictionaries by executing ALTER TYPE mydict ADD ENTRY. (This
also may answer your comment regarding a limit on SQL statement size.)
4. The compression scheme can be altered in the future if needed.
Every compressed document stores algorithm_version (1 byte).
Does this plan of action sound OK to you? At this point it is not too
difficult to make design changes.
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2022-06-28 12:37 UTC | newest]
Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-11-11 22:40 [PATCH 1/1] resownerbench Heikki Linnakangas <[email protected]>
2022-04-22 08:30 [PATCH] Compression dictionaries for JSONB Aleksander Alekseev <[email protected]>
2022-06-01 21:29 ` Re: [PATCH] Compression dictionaries for JSONB Jacob Champion <[email protected]>
2022-06-02 13:30 ` Re: [PATCH] Compression dictionaries for JSONB Aleksander Alekseev <[email protected]>
2022-06-02 22:21 ` Re: [PATCH] Compression dictionaries for JSONB Jacob Champion <[email protected]>
2022-06-23 15:48 ` Re: [PATCH] Compression dictionaries for JSONB Simon Riggs <[email protected]>
2022-06-28 12:37 ` Re: [PATCH] Compression dictionaries for JSONB Aleksander Alekseev <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox