public inbox for [email protected]
help / color / mirror / Atom feedFrom: Nikita Malakhov <[email protected]>
To: Michael Paquier <[email protected]>
Cc: Matthias van de Meent <[email protected]>
Cc: Robert Haas <[email protected]>
Cc: Aleksander Alekseev <[email protected]>
Cc: PostgreSQL Hackers <[email protected]>
Subject: Re: RFI: Extending the TOAST Pointer
Date: Wed, 6 Dec 2023 12:49:46 +0300
Message-ID: <CAN-LCVNxbnpHh4PVUUc9g6dPibE8wZALiLtxcs3TjfivxDkCkA@mail.gmail.com> (raw)
In-Reply-To: <[email protected]>
References: <CAN-LCVMq2X=fhx7KLxfeDyb3P+BXuCkHC0g=9GF+JD4izfVa0Q@mail.gmail.com>
<CAJ7c6TNAYyeMYKVkiwOZChy7UpE_CkjpYOk73gcWTXMkLkEyzw@mail.gmail.com>
<CAEze2WibtT+EdGscamqDm3+7qauWi_S6nVUhbK_qg1rkqN_48w@mail.gmail.com>
<CA+TgmoaVcjUkmtWdc_9QjBzvSShjDBYk-5XFNaOvYLgGROjJMA@mail.gmail.com>
<CAEze2Wg6TjMmOAOjaN9-J-rYt4nUT9qdE=DhqwRhj0j0Eyirpw@mail.gmail.com>
<[email protected]>
Hi,
Here's the PoC for a custom TOAST pointer. The main idea is that custom
pointer
provides data space allowing to store custom metadata (i.e. TOAST method,
relation
OIDs, advanced compression information, etc, and even keep part of the data
inline.
Any feedback would be greatly appreciated.
--
Regards,
Nikita Malakhov
Postgres Professional
The Russian Postgres Company
https://postgrespro.ru/
Attachments:
[application/octet-stream] 0001_custom_toast_pointer_v1.patch (5.9K, ../CAN-LCVNxbnpHh4PVUUc9g6dPibE8wZALiLtxcs3TjfivxDkCkA@mail.gmail.com/3-0001_custom_toast_pointer_v1.patch)
download | inline diff:
From 0f0047a9ba12eddbbd4385e8e89ec47b9d87905c Mon Sep 17 00:00:00 2001
From: Nikita Malakhov <[email protected]>
Date: Wed, 6 Dec 2023 12:13:32 +0300
Subject: [PATCH] Introducing custom TOAST pointer, with ability to store
custom defined data inline as well as out-of-line, and custom metadata.
---
src/backend/access/brin/brin_minmax_multi.c | 1 +
src/include/varatt.h | 21 ++++-
src/include/varatt_custom.h | 86 +++++++++++++++++++++
3 files changed, 105 insertions(+), 3 deletions(-)
create mode 100644 src/include/varatt_custom.h
diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c
index 9811451b54..e81ad5cba7 100644
--- a/src/backend/access/brin/brin_minmax_multi.c
+++ b/src/backend/access/brin/brin_minmax_multi.c
@@ -55,6 +55,7 @@
* src/backend/access/brin/brin_minmax_multi.c
*/
#include "postgres.h"
+#include "varatt_custom.h"
/* needed for PGSQL_AF_INET */
#include <sys/socket.h>
diff --git a/src/include/varatt.h b/src/include/varatt.h
index e34870526b..d84c9e96a5 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -86,17 +86,28 @@ typedef enum vartag_external
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
- VARTAG_ONDISK = 18
+ VARTAG_ONDISK = 18,
+ VARTAG_CUSTOM = 127
} vartag_external;
+typedef enum ToastPtrSizeType
+{
+ TPTR_DATUM_SIZE,
+ TPTR_RAW_SIZE,
+ TPTR_STORAGE_SIZE
+} ToastPtrSizeType;
+
+extern Size toast_custom_datum_size(const void *ptr, ToastPtrSizeType sz_type);
+
/* this test relies on the specific tag values above */
#define VARTAG_IS_EXPANDED(tag) \
(((tag) & ~1) == VARTAG_EXPANDED_RO)
-#define VARTAG_SIZE(tag) \
+#define VARTAG_SIZE(tag, ptr) \
((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
(tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
+ (tag) == VARTAG_CUSTOM ? toast_custom_datum_size((const void *)(ptr), TPTR_DATUM_SIZE) : \
(AssertMacro(false), 0))
/*
@@ -253,6 +264,7 @@ typedef struct
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
+#define VARHDRSZ_CUSTOM offsetof(varattrib_1b_e, va_data)
#define VARATT_SHORT_MAX 0x7F
#define VARATT_CAN_MAKE_SHORT(PTR) \
@@ -282,7 +294,7 @@ typedef struct
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR)
-#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
+#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR), (const void *)(PTR)))
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
@@ -302,6 +314,9 @@ typedef struct
#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
+#define VARATT_IS_CUSTOM(PTR) \
+ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_CUSTOM)
+
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
diff --git a/src/include/varatt_custom.h b/src/include/varatt_custom.h
new file mode 100644
index 0000000000..6a66383413
--- /dev/null
+++ b/src/include/varatt_custom.h
@@ -0,0 +1,86 @@
+/*-------------------------------------------------------------------------
+ *
+ * varatt_custom.h
+ * CUSTOM Toast Pointer definition and macros
+ *
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 2016-2023, Postgres Professional
+ *
+ * IDENTIFICATION
+ * contrib/toastapi/varatt_custom.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef VARATT_CUSTOM_H
+#define VARATT_CUSTOM_H
+
+#include "postgres.h"
+#include "varatt.h"
+
+typedef struct uint32align16
+{
+ uint16 hi;
+ uint16 lo;
+} uint32align16;
+
+#define set_uint32align16(p, v) \
+ ( \
+ (p)->hi = (v) >> 16, \
+ (p)->lo = (v) & 0xffff \
+ )
+
+#define get_uint32align16(p) \
+ (((uint32)((p)->hi)) << 16 | ((uint32)((p)->lo)))
+
+/* varatt_custom uses 16bit aligment */
+typedef struct varatt_custom
+{
+ uint32align16 va_tptrdatalen; /* total size of toast pointer, < BLCKSZ */
+ uint32align16 va_rawsize; /* Original data size (includes header) */
+ char va_tptrdata[FLEXIBLE_ARRAY_MEMBER]; /* Custom data */
+} varatt_custom;
+
+/* Custom Toast pointer */
+#define VARATT_CUSTOM_GET_TOASTPOINTER(PTR) \
+ ((varatt_custom *) VARDATA_EXTERNAL(PTR))
+
+#define VARATT_CUSTOM_GET_DATA_RAW_SIZE(PTR) \
+ (get_uint32align16(&VARATT_CUSTOM_GET_TOASTPOINTER(PTR)->va_rawsize))
+
+#define VARATT_CUSTOM_SET_DATA_RAW_SIZE(PTR, V) \
+ (set_uint32align16(&VARATT_CUSTOM_GET_TOASTPOINTER(PTR)->va_rawsize, (V)))
+
+#define VARATT_CUSTOM_GET_DATA_SIZE(PTR) \
+ (get_uint32align16(&VARATT_CUSTOM_GET_TOASTPOINTER(PTR)->va_tptrdatalen))
+
+#define VARATT_CUSTOM_SET_DATA_SIZE(PTR, V) \
+ (set_uint32align16(&VARATT_CUSTOM_GET_TOASTPOINTER(PTR)->va_tptrdatalen, (V)))
+
+#define VARATT_CUSTOM_GET_DATA(PTR) \
+ (VARATT_CUSTOM_GET_TOASTPOINTER(PTR)->va_tptrdata)
+
+#define VARATT_CUSTOM_SIZE(datalen) \
+ ((Size) VARHDRSZ_EXTERNAL + offsetof(varatt_custom, va_tptrdata) + (datalen))
+
+#define VARATT_CUSTOM_MAX_DATA_SIZE \
+ (MaxAllocSize - VARATT_CUSTOM_SIZE(0))
+
+#define VARSIZE_CUSTOM(PTR) VARATT_CUSTOM_SIZE(VARATT_CUSTOM_GET_DATA_SIZE(PTR))
+
+Size
+toast_custom_datum_size(const void *ptr, ToastPtrSizeType sz_type)
+{
+ if (sz_type == TPTR_DATUM_SIZE ||
+ sz_type == TPTR_STORAGE_SIZE)
+ return offsetof(varatt_custom, va_tptrdatalen) + VARATT_CUSTOM_GET_DATA_SIZE(ptr);
+ else if (sz_type == TPTR_RAW_SIZE)
+ return VARATT_CUSTOM_GET_DATA_RAW_SIZE(ptr);
+ else
+ elog(ERROR, "invalid toast_custom_datum_size() size type");
+
+ return 0; /* avoid warning */
+}
+
+#endif
--
2.25.1
view thread (4+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected], [email protected], [email protected]
Subject: Re: RFI: Extending the TOAST Pointer
In-Reply-To: <CAN-LCVNxbnpHh4PVUUc9g6dPibE8wZALiLtxcs3TjfivxDkCkA@mail.gmail.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox