public inbox for [email protected]
help / color / mirror / Atom feedFrom: Dharin Shah <[email protected]>
To: [email protected]
Subject: [PATCH] Add zstd compression for TOAST using extended header format
Date: Mon, 15 Dec 2025 20:16:59 +0100
Message-ID: <CAOj6k6dkseVvZzmEAWvBd6twZsCU0DbN+qeM7CoDuMM3r9doiw@mail.gmail.com> (raw)
In-Reply-To: <CAOj6k6dy2CRVA6Lsb5N59zE-7KNVKt=oYwWyg8ULK8zOOY8e7A@mail.gmail.com>
References: <CAOj6k6dy2CRVA6Lsb5N59zE-7KNVKt=oYwWyg8ULK8zOOY8e7A@mail.gmail.com>
Hello PG Hackers,
Want to submit a patch that implements zstd compression for TOAST data
using a 20-byte TOAST pointer format, directly addressing the concerns
raised in prior discussions [1
<https://www.postgresql.org/message-id/flat/CAFAfj_F4qeRCNCYPk1vgH42fDZpjQWKO%2Bufq3FyoVyUa5AviFA%40m...;
][2
<https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22=5xVBg7S4vr5rQ@mail....;
][3
<https://www.postgresql.org/message-id/flat/[email protected];].
A bit of a background in the 2022 thread [3
<https://www.postgresql.org/message-id/flat/[email protected];],
The overall suggestion was to have something extensible for the TOAST header
i.e. something like:
00 = PGLZ
01 = LZ4
10 = reserved for future emergencies
11 = extended header with additional type byte
This patch implements that idea.
The new header format:
struct varatt_external_extended {
int32 va_rawsize; /* same as legacy */
uint32 va_extinfo; /* cmid=3 signals extended format */
uint8 va_flags; /* feature flags */
uint8 va_data[3]; /* va_data[0] = compression method */
Oid va_valueid; /* same as legacy */
Oid va_toastrelid; /* same as legacy */
};
*A few notes:*
- Zstd only applies to external TOAST, not inline compression. The 2-bit
limit in va_tcinfo stays as-is for inline data, where pglz/lz4 work fine
anyway. Zstd's wins show up on larger values.
- A GUC use_extended_toast_header controls whether pglz/lz4 also use the
20-byte format (defaults to off for compatibility, can enable it if you
want consistency).
- Legacy 16-byte pointers continue to work - we check the vartag to
determine which format to read.
The 4 extra bytes per pointer is negligible for typical TOAST data sizes,
and it gives us room to grow.
Regards,
Dharin
Attachments:
[application/x-patch] zstd-toast-compression-external.patch (78.2K, 3-zstd-toast-compression-external.patch)
download | inline diff:
From fdaae5dc9e9837f73b991100adcba6d76dda1f40 Mon Sep 17 00:00:00 2001
From: Dharin Shah <[email protected]>
Date: Sat, 13 Dec 2025 11:16:35 +0100
Subject: [PATCH] Add zstd compression support for TOAST using extended header
format
---
contrib/amcheck/verify_heapam.c | 69 +++++-
src/backend/access/common/detoast.c | 164 ++++++++++++---
src/backend/access/common/toast_compression.c | 199 +++++++++++++++++-
src/backend/access/common/toast_internals.c | 198 +++++++++++++++--
src/backend/access/table/toast_helper.c | 2 +-
.../replication/logical/reorderbuffer.c | 38 +++-
src/backend/utils/adt/varlena.c | 26 ++-
src/backend/utils/misc/guc_parameters.dat | 7 +-
src/backend/utils/misc/guc_tables.c | 3 +
src/include/access/detoast.h | 41 +++-
src/include/access/toast_compression.h | 36 ++++
src/include/access/toast_internals.h | 10 +-
src/include/varatt.h | 160 +++++++++++++-
src/test/modules/meson.build | 1 +
src/test/modules/test_toast_ext/Makefile | 20 ++
.../expected/test_toast_ext.out | 187 ++++++++++++++++
.../expected/test_toast_ext_1.out | 37 ++++
src/test/modules/test_toast_ext/meson.build | 33 +++
.../test_toast_ext/sql/test_toast_ext.sql | 136 ++++++++++++
.../test_toast_ext/test_toast_ext--1.0.sql | 19 ++
.../modules/test_toast_ext/test_toast_ext.c | 140 ++++++++++++
.../test_toast_ext/test_toast_ext.control | 5 +
22 files changed, 1440 insertions(+), 91 deletions(-)
create mode 100644 src/test/modules/test_toast_ext/Makefile
create mode 100644 src/test/modules/test_toast_ext/expected/test_toast_ext.out
create mode 100644 src/test/modules/test_toast_ext/expected/test_toast_ext_1.out
create mode 100644 src/test/modules/test_toast_ext/meson.build
create mode 100644 src/test/modules/test_toast_ext/sql/test_toast_ext.sql
create mode 100644 src/test/modules/test_toast_ext/test_toast_ext--1.0.sql
create mode 100644 src/test/modules/test_toast_ext/test_toast_ext.c
create mode 100644 src/test/modules/test_toast_ext/test_toast_ext.control
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 130b3533463..25cae4d0380 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1665,6 +1665,8 @@ check_tuple_attribute(HeapCheckContext *ctx)
uint16 infomask;
CompactAttribute *thisatt;
struct varatt_external toast_pointer;
+ struct varatt_external_extended toast_pointer_ext;
+ bool is_extended;
infomask = ctx->tuphdr->t_infomask;
thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
@@ -1717,13 +1719,14 @@ check_tuple_attribute(HeapCheckContext *ctx)
/*
* Check that VARTAG_SIZE won't hit an Assert on a corrupt va_tag before
- * risking a call into att_addlength_pointer
+ * risking a call into att_addlength_pointer. Both legacy (VARTAG_ONDISK)
+ * and extended (VARTAG_ONDISK_EXTENDED) on-disk formats are valid.
*/
if (VARATT_IS_EXTERNAL(tp + ctx->offset))
{
uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
- if (va_tag != VARTAG_ONDISK)
+ if (va_tag != VARTAG_ONDISK && va_tag != VARTAG_ONDISK_EXTENDED)
{
report_corruption(ctx,
psprintf("toasted attribute has unexpected TOAST tag %u",
@@ -1768,9 +1771,23 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* It is external, and we're looking at a page on disk */
/*
- * Must copy attr into toast_pointer for alignment considerations
+ * Must copy attr into toast_pointer for alignment considerations.
+ * Handle both legacy (VARTAG_ONDISK) and extended (VARTAG_ONDISK_EXTENDED)
+ * formats.
*/
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ is_extended = (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED);
+
+ if (is_extended)
+ {
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ /* Copy common fields for simpler code below */
+ toast_pointer.va_rawsize = toast_pointer_ext.va_rawsize;
+ toast_pointer.va_extinfo = toast_pointer_ext.va_extinfo;
+ toast_pointer.va_valueid = toast_pointer_ext.va_valueid;
+ toast_pointer.va_toastrelid = toast_pointer_ext.va_toastrelid;
+ }
+ else
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/* Toasted attributes too large to be untoasted should never be stored */
if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
@@ -1785,8 +1802,11 @@ check_tuple_attribute(HeapCheckContext *ctx)
ToastCompressionId cmid;
bool valid = false;
- /* Compressed attributes should have a valid compression method */
- cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+ /*
+ * Compressed attributes should have a valid compression method.
+ * For extended pointers with cmid==3, the actual method is in va_data[0].
+ */
+ cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
switch (cmid)
{
/* List of all valid compression method IDs */
@@ -1795,6 +1815,27 @@ check_tuple_attribute(HeapCheckContext *ctx)
valid = true;
break;
+ /* Extended compression (zstd or pglz/lz4 in extended format) */
+ case TOAST_EXTENDED_COMPRESSION_ID:
+ if (is_extended)
+ {
+ uint8 ext_method = VARATT_EXTERNAL_GET_EXT_COMPRESSION_METHOD(toast_pointer_ext);
+
+ /* Validate extended compression method */
+ switch (ext_method)
+ {
+ case TOAST_PGLZ_EXT_METHOD:
+ case TOAST_LZ4_EXT_METHOD:
+ case TOAST_ZSTD_EXT_METHOD:
+ valid = true;
+ break;
+ default:
+ /* Invalid extended method will be reported below */
+ break;
+ }
+ }
+ break;
+
/* Recognized but invalid compression method ID */
case TOAST_INVALID_COMPRESSION_ID:
break;
@@ -1840,7 +1881,21 @@ check_tuple_attribute(HeapCheckContext *ctx)
ta = palloc0_object(ToastedAttribute);
- VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+ /*
+ * Extract toast pointer based on format. For extended format,
+ * copy common fields from toast_pointer which we already extracted
+ * above.
+ */
+ if (is_extended)
+ {
+ ta->toast_pointer.va_rawsize = toast_pointer.va_rawsize;
+ ta->toast_pointer.va_extinfo = toast_pointer.va_extinfo;
+ ta->toast_pointer.va_valueid = toast_pointer.va_valueid;
+ ta->toast_pointer.va_toastrelid = toast_pointer.va_toastrelid;
+ }
+ else
+ VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+
ta->blkno = ctx->blkno;
ta->offnum = ctx->offnum;
ta->attnum = ctx->attnum;
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 62651787742..6d1c08900e8 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -16,6 +16,7 @@
#include "access/detoast.h"
#include "access/table.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/toast_internals.h"
#include "common/int.h"
#include "common/pg_lzcompress.h"
@@ -225,12 +226,47 @@ detoast_attr_slice(struct varlena *attr,
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- struct varatt_external toast_pointer;
+ int32 max_size;
+ bool is_compressed;
+ bool is_pglz = false;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Handle both legacy 16-byte and extended 20-byte on-disk TOAST
+ * pointers. Check the vartag to determine which format.
+ */
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED)
+ {
+ struct varatt_external_extended toast_pointer_ext;
+ uint8 ext_method;
+
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ max_size = VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(toast_pointer_ext);
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(toast_pointer_ext);
+
+ /* Check if this is pglz for slice optimization */
+ if (is_compressed &&
+ VARATT_EXTERNAL_HAS_FLAG(toast_pointer_ext, TOAST_EXT_FLAG_COMPRESSION))
+ {
+ ext_method = VARATT_EXTERNAL_GET_EXT_COMPRESSION_METHOD(toast_pointer_ext);
+ is_pglz = (ext_method == TOAST_PGLZ_EXT_METHOD);
+ }
+ }
+ else
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer);
+
+ /* Check if this is pglz for slice optimization */
+ if (is_compressed)
+ is_pglz = (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
+ TOAST_PGLZ_COMPRESSION_ID);
+ }
/* fast path for non-compressed external datums */
- if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (!is_compressed)
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/*
@@ -240,19 +276,16 @@ detoast_attr_slice(struct varlena *attr,
*/
if (slicelimit >= 0)
{
- int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
-
/*
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or zstd data, we'll have to fetch
+ * the whole thing, because there doesn't seem to be an API call
+ * to determine how much compressed data we need to be sure of
+ * being able to decompress the required slice.
*/
- if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
- TOAST_PGLZ_COMPRESSION_ID)
+ if (is_pglz)
max_size = pglz_maximum_compressed_size(slicelimit, max_size);
/*
@@ -344,20 +377,42 @@ toast_fetch_datum(struct varlena *attr)
{
Relation toastrel;
struct varlena *result;
- struct varatt_external toast_pointer;
int32 attrsize;
+ Oid toastrelid;
+ Oid valueid;
+ bool is_compressed;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Handle both legacy 16-byte and extended 20-byte on-disk TOAST pointers.
+ * Check the vartag to determine which format we're dealing with.
+ */
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED)
+ {
+ struct varatt_external_extended toast_pointer_ext;
+
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(toast_pointer_ext);
+ toastrelid = toast_pointer_ext.va_toastrelid;
+ valueid = toast_pointer_ext.va_valueid;
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(toast_pointer_ext);
+ }
+ else
+ {
+ struct varatt_external toast_pointer;
- attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = toast_pointer.va_valueid;
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer);
+ }
result = (struct varlena *) palloc(attrsize + VARHDRSZ);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
else
SET_VARSIZE(result, attrsize + VARHDRSZ);
@@ -369,10 +424,10 @@ toast_fetch_datum(struct varlena *attr)
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, 0, attrsize, result);
/* Close toast table */
@@ -398,23 +453,45 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
{
Relation toastrel;
struct varlena *result;
- struct varatt_external toast_pointer;
int32 attrsize;
+ Oid toastrelid;
+ Oid valueid;
+ bool is_compressed;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Handle both legacy 16-byte and extended 20-byte on-disk TOAST pointers.
+ * Check the vartag to determine which format we're dealing with.
+ */
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED)
+ {
+ struct varatt_external_extended toast_pointer_ext;
+
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(toast_pointer_ext);
+ toastrelid = toast_pointer_ext.va_toastrelid;
+ valueid = toast_pointer_ext.va_valueid;
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(toast_pointer_ext);
+ }
+ else
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = toast_pointer.va_valueid;
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer);
+ }
/*
* It's nonsense to fetch slices of a compressed datum unless when it's a
* prefix -- this isn't lo_* we can't return a compressed datum which is
* meaningful to toast later.
*/
- Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
-
- attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ Assert(!is_compressed || 0 == sliceoffset);
if (sliceoffset >= attrsize)
{
@@ -427,7 +504,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
* space required by va_tcinfo, which is stored at the beginning as an
* int32 value.
*/
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0)
+ if (is_compressed && slicelength > 0)
slicelength = slicelength + sizeof(int32);
/*
@@ -440,7 +517,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
result = (struct varlena *) palloc(slicelength + VARHDRSZ);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
else
SET_VARSIZE(result, slicelength + VARHDRSZ);
@@ -449,10 +526,10 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
return result; /* Can save a lot of work at this point! */
/* Open the toast relation */
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, sliceoffset, slicelength,
result);
@@ -485,6 +562,9 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_EXTENDED_COMPRESSION_ID:
+ /* zstd-compressed data */
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +608,9 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_EXTENDED_COMPRESSION_ID:
+ /* zstd-compressed data */
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -549,11 +632,15 @@ toast_raw_datum_size(Datum value)
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- /* va_rawsize is the size of the original datum -- including header */
- struct varatt_external toast_pointer;
+ /*
+ * va_rawsize is the size of the original datum -- including header.
+ * It's at offset 0 in both varatt_external and varatt_external_extended,
+ * so we can read just the first 4 bytes regardless of format.
+ */
+ int32 va_rawsize;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_rawsize;
+ memcpy(&va_rawsize, VARDATA_EXTERNAL(attr), sizeof(va_rawsize));
+ result = va_rawsize;
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
@@ -609,11 +696,18 @@ toast_datum_size(Datum value)
* Attribute is stored externally - return the extsize whether
* compressed or not. We do not count the size of the toast pointer
* ... should we?
+ *
+ * va_extinfo is at offset 4 in both varatt_external and
+ * varatt_external_extended, so we can read the first 8 bytes
+ * regardless of format.
*/
- struct varatt_external toast_pointer;
+ struct {
+ int32 va_rawsize;
+ uint32 va_extinfo;
+ } common;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ memcpy(&common, VARDATA_EXTERNAL(attr), sizeof(common));
+ result = common.va_extinfo & VARLENA_EXTSIZE_MASK;
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 926f1e4008a..422e2c5967a 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -17,13 +17,19 @@
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
+
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
+#include "utils/memutils.h"
#include "varatt.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
+bool use_extended_toast_header = false;
#define NO_COMPRESSION_SUPPORT(method) \
ereport(ERROR, \
@@ -249,11 +255,16 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
* Extract compression ID from a varlena.
*
* Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed.
+ *
+ * For external data stored in extended format (VARTAG_ONDISK_EXTENDED),
+ * the actual compression method is stored in va_data[0]. We map that
+ * back to the appropriate ToastCompressionId for legacy compatibility.
*/
ToastCompressionId
toast_get_compression_id(struct varlena *attr)
{
ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
+ vartag_external tag;
/*
* If it is stored externally then fetch the compression method id from
@@ -262,12 +273,52 @@ toast_get_compression_id(struct varlena *attr)
*/
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- struct varatt_external toast_pointer;
-
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
-
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
- cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
+ tag = VARTAG_EXTERNAL(attr);
+ if (tag == VARTAG_ONDISK)
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
+ }
+ else
+ {
+ struct varatt_external_extended toast_pointer_ext;
+ uint8 ext_method;
+
+ Assert(tag == VARTAG_ONDISK_EXTENDED);
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+
+ if (VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(toast_pointer_ext))
+ {
+ /*
+ * Extended format stores the actual method in va_data[0].
+ * Map it back to ToastCompressionId for reporting purposes.
+ */
+ ext_method = VARATT_EXTERNAL_GET_EXT_COMPRESSION_METHOD(toast_pointer_ext);
+ switch (ext_method)
+ {
+ case TOAST_PGLZ_EXT_METHOD:
+ cmid = TOAST_PGLZ_COMPRESSION_ID;
+ break;
+ case TOAST_LZ4_EXT_METHOD:
+ cmid = TOAST_LZ4_COMPRESSION_ID;
+ break;
+ case TOAST_ZSTD_EXT_METHOD:
+ cmid = TOAST_EXTENDED_COMPRESSION_ID;
+ break;
+ case TOAST_UNCOMPRESSED_EXT_METHOD:
+ /* Uncompressed data in extended format */
+ cmid = TOAST_INVALID_COMPRESSION_ID;
+ break;
+ default:
+ elog(ERROR, "invalid extended compression method %d",
+ ext_method);
+ }
+ }
+ }
}
else if (VARATT_IS_COMPRESSED(attr))
cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
@@ -275,6 +326,133 @@ toast_get_compression_id(struct varlena *attr)
return cmid;
}
+/*
+ * Zstandard (zstd) compression/decompression for TOAST (extended methods).
+ *
+ * These routines use the same basic shape as the pglz and LZ4 helpers,
+ * but are only available when PostgreSQL is built with USE_ZSTD.
+ */
+
+/*
+ * Compress a varlena using ZSTD.
+ *
+ * Returns the compressed varlena, or NULL if compression fails or does
+ * not save space.
+ */
+static struct varlena *
+zstd_compress_datum_internal(const struct varlena *value, int level)
+{
+#ifndef USE_ZSTD
+ NO_COMPRESSION_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+ Size valsize;
+ Size max_size;
+ Size out_size;
+ struct varlena *tmp;
+ size_t rc;
+
+ valsize = VARSIZE_ANY_EXHDR(value);
+
+ /*
+ * Compute an upper bound for the compressed size and allocate enough
+ * space for the compressed payload plus the varlena header.
+ */
+ max_size = ZSTD_compressBound(valsize);
+ if (max_size > (Size) (MaxAllocSize - VARHDRSZ_COMPRESSED))
+ ereport(ERROR,
+ (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+ errmsg("compressed data would exceed maximum allocation size")));
+
+ tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
+
+ rc = ZSTD_compress((char *) tmp + VARHDRSZ_COMPRESSED, max_size,
+ VARDATA_ANY(value), valsize, level);
+ if (ZSTD_isError(rc))
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("zstd compression failed: %s",
+ ZSTD_getErrorName(rc))));
+
+ out_size = (Size) rc;
+
+ /*
+ * If the compressed representation is not smaller than the original
+ * payload, give up and return NULL so that callers can fall back to
+ * storing the datum uncompressed or with a different method.
+ */
+ if (out_size >= valsize)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, out_size + VARHDRSZ_COMPRESSED);
+
+ return tmp;
+#endif /* USE_ZSTD */
+}
+
+struct varlena *
+zstd_compress_datum(const struct varlena *value)
+{
+#ifndef USE_ZSTD
+ NO_COMPRESSION_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+ return zstd_compress_datum_internal(value, ZSTD_CLEVEL_DEFAULT);
+#endif
+}
+
+/*
+ * Decompress a varlena that was compressed using ZSTD.
+ */
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifndef USE_ZSTD
+ NO_COMPRESSION_SUPPORT("zstd");
+ return NULL; /* keep compiler quiet */
+#else
+ struct varlena *result;
+ Size rawsize;
+ size_t rc;
+
+ /* allocate memory for the uncompressed data */
+ rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(value);
+ result = (struct varlena *) palloc(rawsize + VARHDRSZ);
+
+ rc = ZSTD_decompress(VARDATA(result), rawsize,
+ (char *) value + VARHDRSZ_COMPRESSED,
+ VARSIZE(value) - VARHDRSZ_COMPRESSED);
+ if (ZSTD_isError(rc) || rc != rawsize)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed zstd data is corrupt or truncated")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif /* USE_ZSTD */
+}
+
+/*
+ * Decompress part of a varlena that was compressed using ZSTD.
+ *
+ * At least initially we don't try to be clever with streaming slice
+ * decompression here; instead we just decompress the full datum and
+ * let higher layers perform the slicing. Callers should prefer the
+ * regular zstd_decompress_datum() when they know they need the whole
+ * value anyway.
+ */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+ /* For now, just fall back to full decompression. */
+ (void) slicelength;
+ return zstd_decompress_datum(value);
+}
+
/*
* CompressionNameToMethod - Get compression method from compression name
*
@@ -293,6 +471,13 @@ CompressionNameToMethod(const char *compression)
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd") == 0)
+ {
+#ifndef USE_ZSTD
+ NO_COMPRESSION_SUPPORT("zstd");
+#endif
+ return TOAST_ZSTD_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -309,6 +494,8 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_COMPRESSION:
+ return "zstd";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index d06af82de15..039ccc42249 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -18,6 +18,7 @@
#include "access/heapam.h"
#include "access/heaptoast.h"
#include "access/table.h"
+#include "access/toast_compression.h"
#include "access/toast_internals.h"
#include "access/xact.h"
#include "catalog/catalog.h"
@@ -71,6 +72,9 @@ toast_compress_datum(Datum value, char cmethod)
tmp = lz4_compress_datum((const struct varlena *) DatumGetPointer(value));
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_COMPRESSION:
+ /* zstd uses external storage only; handled by toast_save_datum */
+ return PointerGetDatum(NULL);
default:
elog(ERROR, "invalid compression method %c", cmethod);
}
@@ -113,11 +117,13 @@ toast_compress_datum(Datum value, char cmethod)
* value: datum to be pushed to toast storage
* oldexternal: if not NULL, toast pointer previously representing the datum
* options: options to be passed to heap_insert() for toast rows
+ * cmethod: compression method to use for uncompressed data
* ----------
*/
Datum
toast_save_datum(Relation rel, Datum value,
- struct varlena *oldexternal, int options)
+ struct varlena *oldexternal, int options,
+ char cmethod)
{
Relation toastrel;
Relation *toastidxs;
@@ -125,12 +131,16 @@ toast_save_datum(Relation rel, Datum value,
CommandId mycid = GetCurrentCommandId(true);
struct varlena *result;
struct varatt_external toast_pointer;
+ struct varatt_external_extended toast_pointer_ext;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
int num_indexes;
int validIndex;
+ bool use_extended = false;
+ uint8 ext_method = 0;
+ struct varlena *compressed_to_free = NULL; /* track allocated buffer */
Assert(!VARATT_IS_EXTERNAL(dval));
@@ -167,23 +177,99 @@ toast_save_datum(Relation rel, Datum value,
}
else if (VARATT_IS_COMPRESSED(dval))
{
+ ToastCompressionId cmid;
+
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
+ /* Get compression method from compressed datum */
+ cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+
+ /* Decide whether to use extended 20-byte or legacy 16-byte format */
+ if (cmid == TOAST_EXTENDED_COMPRESSION_ID)
+ {
+ use_extended = true;
+ ext_method = TOAST_ZSTD_EXT_METHOD;
+ }
+ else if (use_extended_toast_header)
+ {
+ /* Use extended format for pglz/lz4 when GUC is enabled */
+ use_extended = true;
+ switch (cmid)
+ {
+ case TOAST_PGLZ_COMPRESSION_ID:
+ ext_method = TOAST_PGLZ_EXT_METHOD;
+ break;
+ case TOAST_LZ4_COMPRESSION_ID:
+ ext_method = TOAST_LZ4_EXT_METHOD;
+ break;
+ default:
+ /* Should not happen, but fall back to legacy format */
+ use_extended = false;
+ break;
+ }
+ }
+
/* set external size and compression method */
- VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
- VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+ if (use_extended)
+ VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
+ VARATT_EXTERNAL_EXTENDED_CMID);
+ else
+ VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, cmid);
+
/* Assert that the numbers look like it's compressed */
Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
}
else
{
- data_p = VARDATA(dval);
- data_todo = VARSIZE(dval) - VARHDRSZ;
- toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extinfo = data_todo;
+ /*
+ * Uncompressed data. If the caller specified zstd compression,
+ * try to compress it now before storing to the TOAST table.
+ */
+ if (cmethod == TOAST_ZSTD_COMPRESSION)
+ {
+ struct varlena *compressed;
+ int32 rawsize;
+
+ rawsize = VARSIZE_ANY_EXHDR((const struct varlena *) dval);
+ compressed = zstd_compress_datum((const struct varlena *) dval);
+ if (compressed != NULL)
+ {
+ /* Set compression method in va_tcinfo */
+ TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(compressed, rawsize,
+ TOAST_EXTENDED_COMPRESSION_ID);
+
+ /* Compression succeeded - use the compressed data */
+ compressed_to_free = compressed; /* track for cleanup */
+ dval = (Pointer) compressed;
+ data_p = VARDATA(compressed);
+ data_todo = VARSIZE(compressed) - VARHDRSZ;
+ toast_pointer.va_rawsize = rawsize + VARHDRSZ;
+
+ /* Use extended format for zstd */
+ use_extended = true;
+ ext_method = TOAST_ZSTD_EXT_METHOD;
+ VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
+ VARATT_EXTERNAL_EXTENDED_CMID);
+ }
+ else
+ {
+ /* Compression failed or didn't save space - store uncompressed */
+ data_p = VARDATA(dval);
+ data_todo = VARSIZE(dval) - VARHDRSZ;
+ toast_pointer.va_rawsize = VARSIZE(dval);
+ toast_pointer.va_extinfo = data_todo;
+ }
+ }
+ else
+ {
+ data_p = VARDATA(dval);
+ data_todo = VARSIZE(dval) - VARHDRSZ;
+ toast_pointer.va_rawsize = VARSIZE(dval);
+ toast_pointer.va_extinfo = data_todo;
+ }
}
/*
@@ -225,15 +311,36 @@ toast_save_datum(Relation rel, Datum value,
toast_pointer.va_valueid = InvalidOid;
if (oldexternal != NULL)
{
- struct varatt_external old_toast_pointer;
+ Oid old_toastrelid;
+ Oid old_valueid;
Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
- if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
+
+ /*
+ * Extract toastrelid and valueid from the old pointer.
+ * Handle both legacy 16-byte and extended 20-byte formats.
+ */
+ if (VARTAG_EXTERNAL(oldexternal) == VARTAG_ONDISK_EXTENDED)
+ {
+ struct varatt_external_extended old_toast_pointer_ext;
+
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(old_toast_pointer_ext, oldexternal);
+ old_toastrelid = old_toast_pointer_ext.va_toastrelid;
+ old_valueid = old_toast_pointer_ext.va_valueid;
+ }
+ else
+ {
+ struct varatt_external old_toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
+ old_toastrelid = old_toast_pointer.va_toastrelid;
+ old_valueid = old_toast_pointer.va_valueid;
+ }
+
+ if (old_toastrelid == rel->rd_toastoid)
{
/* This value came from the old toast table; reuse its OID */
- toast_pointer.va_valueid = old_toast_pointer.va_valueid;
+ toast_pointer.va_valueid = old_valueid;
/*
* There is a corner case here: the table rewrite might have
@@ -348,6 +455,10 @@ toast_save_datum(Relation rel, Datum value,
data_p += chunk_size;
}
+ /* Free compressed buffer if we allocated one */
+ if (compressed_to_free != NULL)
+ pfree(compressed_to_free);
+
/*
* Done - close toast relation and its indexes but keep the lock until
* commit, so as a concurrent reindex done directly on the toast relation
@@ -356,12 +467,35 @@ toast_save_datum(Relation rel, Datum value,
toast_close_indexes(toastidxs, num_indexes, NoLock);
table_close(toastrel, NoLock);
- /*
- * Create the TOAST pointer value that we'll return
- */
- result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
- SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK);
- memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ /* Create the TOAST pointer value that we'll return */
+ if (use_extended)
+ {
+ /*
+ * Build extended TOAST pointer. Copy the common fields from
+ * toast_pointer, then set the extended-format-specific fields.
+ */
+ toast_pointer_ext.va_rawsize = toast_pointer.va_rawsize;
+ toast_pointer_ext.va_extinfo = toast_pointer.va_extinfo;
+ toast_pointer_ext.va_valueid = toast_pointer.va_valueid;
+ toast_pointer_ext.va_toastrelid = toast_pointer.va_toastrelid;
+
+ /* Set extended format fields */
+ toast_pointer_ext.va_flags = TOAST_EXT_FLAG_COMPRESSION;
+ toast_pointer_ext.va_data[0] = ext_method;
+ toast_pointer_ext.va_data[1] = 0;
+ toast_pointer_ext.va_data[2] = 0;
+
+ result = (struct varlena *) palloc(TOAST_POINTER_SIZE_EXTENDED);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_EXTENDED);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer_ext, sizeof(toast_pointer_ext));
+ }
+ else
+ {
+ /* Standard 16-byte TOAST pointer */
+ result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
return PointerGetDatum(result);
}
@@ -377,6 +511,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
{
struct varlena *attr = (struct varlena *) DatumGetPointer(value);
struct varatt_external toast_pointer;
+ struct varatt_external_extended toast_pointer_ext;
Relation toastrel;
Relation *toastidxs;
ScanKeyData toastkey;
@@ -384,17 +519,36 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
HeapTuple toasttup;
int num_indexes;
int validIndex;
+ Oid toastrelid;
+ Oid valueid;
+ bool is_extended;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
return;
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Must copy to access aligned fields. Handle both legacy (16-byte) and
+ * extended (20-byte) on-disk TOAST pointers based on the tag.
+ */
+ is_extended = (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED);
+
+ if (!is_extended)
+ {
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = toast_pointer.va_valueid;
+ }
+ else
+ {
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ toastrelid = toast_pointer_ext.va_toastrelid;
+ valueid = toast_pointer_ext.va_valueid;
+ }
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
+ toastrel = table_open(toastrelid, RowExclusiveLock);
/* Fetch valid relation used for process */
validIndex = toast_open_indexes(toastrel,
@@ -408,7 +562,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
- ObjectIdGetDatum(toast_pointer.va_valueid));
+ ObjectIdGetDatum(valueid));
/*
* Find all the chunks. (We don't actually care whether we see them in
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index 11f97d65367..21381004ba6 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -261,7 +261,7 @@ toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
attr->tai_colflags |= TOASTCOL_IGNORE;
*value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
- options);
+ options, attr->tai_compression);
if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
pfree(DatumGetPointer(old_value));
attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index f18c6fb52b5..9e83ab5978d 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -5137,11 +5137,17 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
/* va_rawsize is the size of the original datum -- including header */
struct varatt_external toast_pointer;
+ struct varatt_external_extended toast_pointer_ext;
struct varatt_indirect redirect_pointer;
struct varlena *new_datum = NULL;
struct varlena *reconstructed;
dlist_iter it;
Size data_done = 0;
+ bool is_extended;
+ Oid valueid;
+ int32 rawsize;
+ int32 extsize;
+ bool is_compressed;
if (attr->attisdropped)
continue;
@@ -5161,14 +5167,36 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
if (!VARATT_IS_EXTERNAL(varlena))
continue;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena);
+ /*
+ * Handle both legacy 16-byte and extended 20-byte on-disk TOAST
+ * pointers based on the tag.
+ */
+ is_extended = VARATT_IS_EXTERNAL_ONDISK(varlena) &&
+ (VARTAG_EXTERNAL(varlena) == VARTAG_ONDISK_EXTENDED);
+
+ if (is_extended)
+ {
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, varlena);
+ valueid = toast_pointer_ext.va_valueid;
+ rawsize = toast_pointer_ext.va_rawsize;
+ extsize = VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(toast_pointer_ext);
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(toast_pointer_ext);
+ }
+ else
+ {
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena);
+ valueid = toast_pointer.va_valueid;
+ rawsize = toast_pointer.va_rawsize;
+ extsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ is_compressed = VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer);
+ }
/*
* Check whether the toast tuple changed, replace if so.
*/
ent = (ReorderBufferToastEnt *)
hash_search(txn->toast_hash,
- &toast_pointer.va_valueid,
+ &valueid,
HASH_FIND,
NULL);
if (ent == NULL)
@@ -5179,7 +5207,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
free[natt] = true;
- reconstructed = palloc0(toast_pointer.va_rawsize);
+ reconstructed = palloc0(rawsize);
ent->reconstructed = reconstructed;
@@ -5204,10 +5232,10 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer));
+ Assert(data_done == extsize);
/* make sure its marked as compressed or not */
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
else
SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index baa5b44ea8d..71a410dc617 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -4206,6 +4206,10 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_EXTENDED_COMPRESSION_ID:
+ /* Extended format currently only supports zstd */
+ result = "zstd";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
@@ -4222,7 +4226,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
{
int typlen;
struct varlena *attr;
- struct varatt_external toast_pointer;
+ Oid valueid;
/* On first call, get the input type's typlen, and save at *fn_extra */
if (fcinfo->flinfo->fn_extra == NULL)
@@ -4249,9 +4253,25 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
PG_RETURN_NULL();
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Handle both legacy 16-byte and extended 20-byte on-disk TOAST pointers.
+ */
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_EXTENDED)
+ {
+ struct varatt_external_extended toast_pointer_ext;
+
+ VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr);
+ valueid = toast_pointer_ext.va_valueid;
+ }
+ else
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ valueid = toast_pointer.va_valueid;
+ }
- PG_RETURN_OID(toast_pointer.va_valueid);
+ PG_RETURN_OID(valueid);
}
/*
diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat
index 3b9d8349078..38c68d1d0a6 100644
--- a/src/backend/utils/misc/guc_parameters.dat
+++ b/src/backend/utils/misc/guc_parameters.dat
@@ -738,7 +738,6 @@
boot_val => 'TOAST_PGLZ_COMPRESSION',
options => 'default_toast_compression_options',
},
-
{ name => 'default_transaction_deferrable', type => 'bool', context => 'PGC_USERSET', group => 'CLIENT_CONN_STATEMENT',
short_desc => 'Sets the default deferrable status of new transactions.',
variable => 'DefaultXactDeferrable',
@@ -3175,6 +3174,12 @@
boot_val => 'DEFAULT_UPDATE_PROCESS_TITLE',
},
+{ name => 'use_extended_toast_header', type => 'bool', context => 'PGC_USERSET', group => 'CLIENT_CONN_STATEMENT',
+ short_desc => 'Use 20-byte extended TOAST header format (required for zstd).',
+ variable => 'use_extended_toast_header',
+ boot_val => 'false',
+},
+
{ name => 'vacuum_buffer_usage_limit', type => 'int', context => 'PGC_USERSET', group => 'RESOURCES_MEM',
short_desc => 'Sets the buffer pool size for VACUUM, ANALYZE, and autovacuum.',
flags => 'GUC_UNIT_KB',
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index f87b558c2c6..f6c09260f1a 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -460,6 +460,9 @@ static const struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd", TOAST_ZSTD_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index e603a2276c3..e591a59569b 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -14,25 +14,58 @@
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
- * into a local "struct varatt_external" toast pointer. This should be
- * just a memcpy, but some versions of gcc seem to produce broken code
- * that assumes the datum contents are aligned. Introducing an explicit
- * intermediate "varattrib_1b_e *" variable seems to fix it.
+ * into a local "struct varatt_external" toast pointer.
+ *
+ * This currently supports only the legacy on-disk TOAST pointer format,
+ * which has VARTAG_ONDISK and a payload size of sizeof(varatt_external).
+ * Extended on-disk pointers (VARTAG_ONDISK_EXTENDED) must be accessed via
+ * VARATT_EXTERNAL_GET_POINTER_EXTENDED().
+ *
+ * This should be just a memcpy, but some versions of gcc seem to produce
+ * broken code that assumes the datum contents are aligned. Introducing
+ * an explicit intermediate "varattrib_1b_e *" variable seems to fix it.
*/
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
do { \
varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
Assert(VARATT_IS_EXTERNAL(attre)); \
+ Assert(VARTAG_EXTERNAL(attre) == VARTAG_ONDISK); \
Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)
+/*
+ * Variant of VARATT_EXTERNAL_GET_POINTER for the extended on-disk TOAST
+ * pointer format. Callers should only use this when they have already
+ * established that the tag is VARTAG_ONDISK_EXTENDED.
+ */
+#define VARATT_EXTERNAL_GET_POINTER_EXTENDED(toast_pointer_ext, attr) \
+do { \
+ varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
+ Assert(VARATT_IS_EXTERNAL(attre)); \
+ Assert(VARTAG_EXTERNAL(attre) == VARTAG_ONDISK_EXTENDED); \
+ Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer_ext) + VARHDRSZ_EXTERNAL); \
+ memcpy(&(toast_pointer_ext), VARDATA_EXTERNAL(attre), sizeof(toast_pointer_ext)); \
+} while (0)
+
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
/* Size of an EXTERNAL datum that contains an indirection pointer */
#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
+/* Size of an EXTERNAL datum that contains an extended TOAST pointer */
+#define TOAST_POINTER_SIZE_EXTENDED (VARHDRSZ_EXTERNAL + sizeof(varatt_external_extended))
+
+/* Validation helpers for TOAST pointer sizes */
+#define TOAST_POINTER_SIZE_IS_VALID(size) \
+ ((size) == TOAST_POINTER_SIZE || \
+ (size) == TOAST_POINTER_SIZE_EXTENDED || \
+ (size) == INDIRECT_POINTER_SIZE)
+
+#define TOAST_POINTER_IS_EXTENDED_SIZE(size) \
+ ((size) == TOAST_POINTER_SIZE_EXTENDED)
+
/* ----------
* detoast_external_attr() -
*
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 13c4612ceed..b769d1bc72d 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -13,14 +13,21 @@
#ifndef TOAST_COMPRESSION_H
#define TOAST_COMPRESSION_H
+#include "varatt.h"
+
/*
* GUC support.
*
* default_toast_compression is an integer for purposes of the GUC machinery,
* but the value is one of the char values defined below, as they appear in
* pg_attribute.attcompression, e.g. TOAST_PGLZ_COMPRESSION.
+ *
+ * use_extended_toast_header controls whether to use the 20-byte extended
+ * TOAST pointer format (required for zstd) instead of the legacy 16-byte
+ * format. When false, zstd compression falls back to pglz.
*/
extern PGDLLIMPORT int default_toast_compression;
+extern PGDLLIMPORT bool use_extended_toast_header;
/*
* Built-in compression method ID. The toast compression header will store
@@ -39,6 +46,7 @@ typedef enum ToastCompressionId
TOAST_PGLZ_COMPRESSION_ID = 0,
TOAST_LZ4_COMPRESSION_ID = 1,
TOAST_INVALID_COMPRESSION_ID = 2,
+ TOAST_EXTENDED_COMPRESSION_ID = 3, /* extended format for future methods */
} ToastCompressionId;
/*
@@ -48,6 +56,7 @@ typedef enum ToastCompressionId
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_COMPRESSION 'z'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
@@ -65,9 +74,36 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* zstd compression/decompression routines (extended methods) */
+extern struct varlena *zstd_compress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value,
+ int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
extern const char *GetCompressionMethodName(char method);
+/*
+ * Feature flags for extended TOAST pointers (varatt_external_extended).
+ * These alias VARATT_EXTERNAL_FLAG_* from varatt.h.
+ */
+#define TOAST_EXT_FLAG_COMPRESSION VARATT_EXTERNAL_FLAG_COMPRESSION
+#define TOAST_EXT_FLAG_CHECKSUM VARATT_EXTERNAL_FLAG_CHECKSUM
+
+/*
+ * Extended compression method IDs for use with extended TOAST format.
+ * Stored in va_data[0] when TOAST_EXT_FLAG_COMPRESSION is set.
+ */
+#define TOAST_PGLZ_EXT_METHOD 0
+#define TOAST_LZ4_EXT_METHOD 1
+#define TOAST_ZSTD_EXT_METHOD 2
+#define TOAST_UNCOMPRESSED_EXT_METHOD 3
+
+/* Validation macros for extended format */
+#define ExtendedCompressionMethodIsValid(method) ((method) <= 255)
+#define ExtendedFlagsAreValid(flags) \
+ (((flags) & ~(TOAST_EXT_FLAG_COMPRESSION | TOAST_EXT_FLAG_CHECKSUM)) == 0)
+
#endif /* TOAST_COMPRESSION_H */
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index 06ae8583c1e..d6bc5c4d179 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -36,11 +36,16 @@ typedef struct toast_compress_header
#define TOAST_COMPRESS_METHOD(ptr) \
(((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
+/*
+ * Set the size and compression method in a compressed datum's header.
+ * Accepts TOAST_EXTENDED_COMPRESSION_ID for extended compression methods.
+ */
#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
do { \
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_EXTENDED_COMPRESSION_ID); \
((toast_compress_header *) (ptr))->tcinfo = \
(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
} while (0)
@@ -50,7 +55,8 @@ extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
extern Datum toast_save_datum(Relation rel, Datum value,
- struct varlena *oldexternal, int options);
+ struct varlena *oldexternal, int options,
+ char cmethod);
extern int toast_open_indexes(Relation toastrel,
LOCKMODE lock,
diff --git a/src/include/varatt.h b/src/include/varatt.h
index aeeabf9145b..5f5829a1ec4 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -45,6 +45,23 @@ typedef struct varatt_external
#define VARLENA_EXTSIZE_BITS 30
#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+/*
+ * Compression method ID stored in the 2 high-order bits of va_extinfo.
+ * Value 3 indicates an extended TOAST pointer format (varatt_external_extended).
+ * This constant is also defined in toast_compression.h for use by TOAST code.
+ */
+#define VARATT_EXTERNAL_EXTENDED_CMID 3
+
+/*
+ * Feature flags for extended on-disk TOAST pointers (varatt_external_extended).
+ *
+ * Keep these in varatt.h (not access/toast headers) so low-level code can
+ * safely manipulate the on-disk representation without depending on higher
+ * layers' header include order.
+ */
+#define VARATT_EXTERNAL_FLAG_COMPRESSION 0x01 /* va_data[0] = method ID */
+#define VARATT_EXTERNAL_FLAG_CHECKSUM 0x02 /* va_data[1-2] = checksum */
+
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
* Datum that's stored in memory, not in an external toast relation.
@@ -76,6 +93,26 @@ typedef struct varatt_expanded
ExpandedObjectHeader *eohptr;
} varatt_expanded;
+/*
+ * Extended TOAST pointer, extending varatt_external from 16 to 20 bytes.
+ *
+ * Identified by compression method ID 3 in va_extinfo bits 30-31. The
+ * va_flags field indicates which optional features are enabled; va_data[]
+ * contains feature-specific data (e.g., compression method in va_data[0]).
+ *
+ * Like varatt_external, stored unaligned and requires memcpy for access.
+ */
+typedef struct varatt_external_extended
+{
+ int32 va_rawsize; /* Original data size (includes header) */
+ uint32 va_extinfo; /* External saved size (30 bits) + extended
+ * indicator (2 bits, value = 3) */
+ uint8 va_flags; /* Feature flags indicating enabled extensions */
+ uint8 va_data[3]; /* Extension data - interpretation depends on flags */
+ Oid va_valueid; /* Unique ID of value within TOAST table */
+ Oid va_toastrelid; /* RelID of TOAST table containing it */
+} varatt_external_extended;
+
/*
* Type tag for the various sorts of "TOAST pointer" datums. The peculiar
* value for VARTAG_ONDISK comes from a requirement for on-disk compatibility
@@ -86,7 +123,17 @@ typedef enum vartag_external
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
- VARTAG_ONDISK = 18
+ VARTAG_ONDISK = 18,
+
+ /*
+ * VARTAG_ONDISK_EXTENDED is used for the extended TOAST pointer format,
+ * which increases the on-disk payload from 16 to 20 bytes. The first
+ * 8 bytes (va_rawsize, va_extinfo) are layout-compatible with
+ * struct varatt_external so that existing code inspecting those fields
+ * continues to work. Older PostgreSQL versions do not know about this
+ * tag and therefore must not be used to read clusters that contain it.
+ */
+ VARTAG_ONDISK_EXTENDED = 19
} vartag_external;
/* Is a TOAST pointer either type of expanded-object pointer? */
@@ -97,7 +144,14 @@ VARTAG_IS_EXPANDED(vartag_external tag)
return ((tag & ~1) == VARTAG_EXPANDED_RO);
}
-/* Size of the data part of a "TOAST pointer" datum */
+/*
+ * Size of the data part of a "TOAST pointer" datum.
+ *
+ * For on-disk TOAST pointers we now support two payload sizes:
+ * the original 16-byte format (VARTAG_ONDISK) described by struct
+ * varatt_external, and a 20-byte extended format
+ * (VARTAG_ONDISK_EXTENDED) described by struct varatt_external_extended.
+ */
static inline Size
VARTAG_SIZE(vartag_external tag)
{
@@ -107,6 +161,8 @@ VARTAG_SIZE(vartag_external tag)
return sizeof(varatt_expanded);
else if (tag == VARTAG_ONDISK)
return sizeof(varatt_external);
+ else if (tag == VARTAG_ONDISK_EXTENDED)
+ return sizeof(varatt_external_extended);
else
{
Assert(false);
@@ -360,7 +416,13 @@ VARATT_IS_EXTERNAL(const void *PTR)
static inline bool
VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
{
- return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK;
+ vartag_external tag;
+
+ if (!VARATT_IS_EXTERNAL(PTR))
+ return false;
+
+ tag = VARTAG_EXTERNAL(PTR);
+ return tag == VARTAG_ONDISK || tag == VARTAG_ONDISK_EXTENDED;
}
/* Is varlena datum an indirect pointer? */
@@ -516,11 +578,11 @@ VARATT_EXTERNAL_GET_COMPRESS_METHOD(struct varatt_external toast_pointer)
}
/* Set size and compress method of an externally-stored varlena datum */
-/* This has to remain a macro; beware multiple evaluations! */
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == VARATT_EXTERNAL_EXTENDED_CMID); \
((toast_pointer).va_extinfo = \
(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
} while (0)
@@ -539,4 +601,92 @@ VARATT_EXTERNAL_IS_COMPRESSED(struct varatt_external toast_pointer)
(Size) (toast_pointer.va_rawsize - VARHDRSZ);
}
+/* Macros for extended TOAST pointers (varatt_external_extended) */
+
+/*
+ * Check if a TOAST pointer uses the extended on-disk format.
+ *
+ * Callers must have already verified VARATT_IS_EXTERNAL_ONDISK() before
+ * calling this; here we look only at the compression-method bits embedded
+ * in va_extinfo.
+ */
+static inline bool
+VARATT_EXTERNAL_IS_EXTENDED(struct varatt_external toast_pointer)
+{
+ return VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
+ VARATT_EXTERNAL_EXTENDED_CMID;
+}
+
+/* Get feature flags from extended pointer */
+static inline uint8
+VARATT_EXTERNAL_GET_FLAGS(struct varatt_external_extended toast_pointer_ext)
+{
+ return toast_pointer_ext.va_flags;
+}
+
+/* Set feature flags in extended pointer */
+#define VARATT_EXTERNAL_SET_FLAGS(toast_pointer_ext, flags) \
+ do { \
+ (toast_pointer_ext).va_flags = (flags); \
+ } while (0)
+
+/* Test if a specific flag is set */
+#define VARATT_EXTERNAL_HAS_FLAG(toast_pointer_ext, flag) \
+ (((toast_pointer_ext).va_flags & (flag)) != 0)
+
+/* Get pointer to extension data array */
+#define VARATT_EXTERNAL_GET_EXT_DATA(toast_pointer_ext) \
+ ((toast_pointer_ext).va_data)
+
+/* Get extended compression method (when TOAST_EXT_FLAG_COMPRESSION is set) */
+static inline uint8
+VARATT_EXTERNAL_GET_EXT_COMPRESSION_METHOD(struct varatt_external_extended toast_pointer_ext)
+{
+ return toast_pointer_ext.va_data[0];
+}
+
+/* Set extended compression method */
+#define VARATT_EXTERNAL_SET_EXT_COMPRESSION_METHOD(toast_pointer_ext, method) \
+ do { \
+ (toast_pointer_ext).va_data[0] = (method); \
+ } while (0)
+
+/* Get extsize and compress method from extended pointer (same as standard) */
+static inline Size
+VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(struct varatt_external_extended toast_pointer_ext)
+{
+ return toast_pointer_ext.va_extinfo & VARLENA_EXTSIZE_MASK;
+}
+
+static inline uint32
+VARATT_EXTERNAL_GET_COMPRESS_METHOD_EXTENDED(struct varatt_external_extended toast_pointer_ext)
+{
+ return toast_pointer_ext.va_extinfo >> VARLENA_EXTSIZE_BITS;
+}
+
+/* Set size and extended indicator in va_extinfo */
+#define VARATT_EXTERNAL_SET_SIZE_AND_EXT_FLAGS(toast_pointer_ext, len, flags) \
+ do { \
+ Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
+ (toast_pointer_ext).va_extinfo = \
+ (len) | ((uint32) VARATT_EXTERNAL_EXTENDED_CMID << VARLENA_EXTSIZE_BITS); \
+ (toast_pointer_ext).va_flags = (flags); \
+ memset((toast_pointer_ext).va_data, 0, 3); \
+ } while (0)
+
+/* Convenience macro for setting extended pointer with compression method */
+#define VARATT_EXTERNAL_SET_SIZE_AND_EXT_COMPRESSION(toast_pointer_ext, len, method) \
+ do { \
+ VARATT_EXTERNAL_SET_SIZE_AND_EXT_FLAGS(toast_pointer_ext, len, VARATT_EXTERNAL_FLAG_COMPRESSION); \
+ VARATT_EXTERNAL_SET_EXT_COMPRESSION_METHOD(toast_pointer_ext, method); \
+ } while (0)
+
+/* Test if extended pointer is compressed (same logic as standard) */
+static inline bool
+VARATT_EXTERNAL_IS_COMPRESSED_EXTENDED(struct varatt_external_extended toast_pointer_ext)
+{
+ return VARATT_EXTERNAL_GET_EXTSIZE_EXTENDED(toast_pointer_ext) <
+ (Size) (toast_pointer_ext.va_rawsize - VARHDRSZ);
+}
+
#endif
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 068fd859a8f..9dff119aa22 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -47,6 +47,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('test_toast_ext')
subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
diff --git a/src/test/modules/test_toast_ext/Makefile b/src/test/modules/test_toast_ext/Makefile
new file mode 100644
index 00000000000..5e2409f918c
--- /dev/null
+++ b/src/test/modules/test_toast_ext/Makefile
@@ -0,0 +1,20 @@
+# src/test/modules/test_toast_ext/Makefile
+
+MODULE_big = test_toast_ext
+OBJS = test_toast_ext.o
+
+EXTENSION = test_toast_ext
+DATA = test_toast_ext--1.0.sql
+
+REGRESS = test_toast_ext
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_toast_ext
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_toast_ext/expected/test_toast_ext.out b/src/test/modules/test_toast_ext/expected/test_toast_ext.out
new file mode 100644
index 00000000000..539f4437655
--- /dev/null
+++ b/src/test/modules/test_toast_ext/expected/test_toast_ext.out
@@ -0,0 +1,187 @@
+--
+-- Tests for extended TOAST header structures and zstd compression
+--
+CREATE EXTENSION test_toast_ext;
+-- Use dedicated schema for test isolation
+CREATE SCHEMA test_toast_ext_schema;
+SET search_path TO test_toast_ext_schema, public;
+-- Compile-time validation tests (always run)
+-- These error out on failure, so completing without error = pass
+SELECT test_toast_structure_sizes();
+ test_toast_structure_sizes
+----------------------------
+
+(1 row)
+
+SELECT test_toast_flag_validation();
+ test_toast_flag_validation
+----------------------------
+
+(1 row)
+
+SELECT test_toast_compression_ids();
+ test_toast_compression_ids
+----------------------------
+
+(1 row)
+
+--
+-- Functional tests for zstd TOAST compression
+-- Skip if not built with USE_ZSTD
+--
+SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE
+ name = 'default_toast_compression' \gset
+\if :skip_test
+ \echo '*** skipping TOAST tests with zstd (not supported) ***'
+ \quit
+\endif
+-- Test basic zstd compression
+CREATE TABLE test_zstd_basic (id serial, data text COMPRESSION zstd);
+INSERT INTO test_zstd_basic (data)
+ VALUES (repeat('PostgreSQL zstd TOAST compression test. ', 3000));
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 42) AS data_prefix
+FROM test_zstd_basic;
+ id | compression | data_length | data_prefix
+----+-------------+-------------+--------------------------------------------
+ 1 | zstd | 120000 | PostgreSQL zstd TOAST compression test. Po
+(1 row)
+
+-- Test slice access
+SELECT id, substr(data, 100, 42) AS slice FROM test_zstd_basic;
+ id | slice
+----+--------------------------------------------
+ 1 | ST compression test. PostgreSQL zstd TOAST
+(1 row)
+
+-- Test UPDATE
+UPDATE test_zstd_basic SET data = repeat('Updated zstd data for TOAST test. ', 3000);
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 35) AS data_prefix
+FROM test_zstd_basic;
+ id | compression | data_length | data_prefix
+----+-------------+-------------+-------------------------------------
+ 1 | zstd | 102000 | Updated zstd data for TOAST test. U
+(1 row)
+
+-- Test extended header with pglz
+SET use_extended_toast_header = on;
+CREATE TABLE test_pglz_extended (data text COMPRESSION pglz);
+INSERT INTO test_pglz_extended (data)
+ VALUES (repeat('PGLZ with extended header format. ', 3000));
+SELECT pg_column_compression(data) AS compression,
+ length(data) AS data_length
+FROM test_pglz_extended;
+ compression | data_length
+-------------+-------------
+ pglz | 102000
+(1 row)
+
+SELECT substr(data, 50, 34) AS slice FROM test_pglz_extended;
+ slice
+------------------------------------
+ ded header format. PGLZ with exten
+(1 row)
+
+-- Test data integrity
+CREATE TABLE test_integrity (
+ method text,
+ original_data text,
+ compressed_data text
+);
+INSERT INTO test_integrity VALUES
+ ('pglz', repeat('Integrity test data pattern. ', 2000), NULL),
+ ('zstd', repeat('Integrity test data pattern. ', 2000), NULL);
+CREATE TABLE test_pglz_integrity (data text COMPRESSION pglz);
+CREATE TABLE test_zstd_integrity (data text COMPRESSION zstd);
+INSERT INTO test_pglz_integrity SELECT original_data FROM test_integrity WHERE method = 'pglz';
+INSERT INTO test_zstd_integrity SELECT original_data FROM test_integrity WHERE method = 'zstd';
+SELECT 'pglz' AS method,
+ md5((SELECT original_data FROM test_integrity WHERE method = 'pglz')) =
+ md5((SELECT data FROM test_pglz_integrity)) AS checksum_match;
+ method | checksum_match
+--------+----------------
+ pglz | t
+(1 row)
+
+SELECT 'zstd' AS method,
+ md5((SELECT original_data FROM test_integrity WHERE method = 'zstd')) =
+ md5((SELECT data FROM test_zstd_integrity)) AS checksum_match;
+ method | checksum_match
+--------+----------------
+ zstd | t
+(1 row)
+
+-- Test CLUSTER and VACUUM FULL
+CREATE TABLE test_cluster_zstd (id serial PRIMARY KEY, data text COMPRESSION zstd);
+INSERT INTO test_cluster_zstd (data)
+ VALUES (repeat('Data for CLUSTER test with zstd compression. ', 2500));
+SELECT 'before_cluster' AS stage, md5(data) AS hash FROM test_cluster_zstd;
+ stage | hash
+----------------+----------------------------------
+ before_cluster | b4132e799bbd065a7e9266159aa82dc1
+(1 row)
+
+CLUSTER test_cluster_zstd USING test_cluster_zstd_pkey;
+SELECT 'after_cluster' AS stage,
+ pg_column_compression(data) AS compression,
+ md5(data) AS hash
+FROM test_cluster_zstd;
+ stage | compression | hash
+---------------+-------------+----------------------------------
+ after_cluster | zstd | b4132e799bbd065a7e9266159aa82dc1
+(1 row)
+
+VACUUM FULL test_cluster_zstd;
+SELECT 'after_vacuum_full' AS stage,
+ pg_column_compression(data) AS compression,
+ md5(data) AS hash
+FROM test_cluster_zstd;
+ stage | compression | hash
+-------------------+-------------+----------------------------------
+ after_vacuum_full | zstd | b4132e799bbd065a7e9266159aa82dc1
+(1 row)
+
+-- Test GUC toggling (mixed formats in same table)
+SET use_extended_toast_header = on;
+CREATE TABLE test_guc_toggle (id serial, data text COMPRESSION pglz);
+INSERT INTO test_guc_toggle (data)
+ VALUES (repeat('Data created with extended header on. ', 3000));
+SELECT 'with_ext_on' AS stage,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length
+FROM test_guc_toggle;
+ stage | compression | data_length
+-------------+-------------+-------------
+ with_ext_on | pglz | 114000
+(1 row)
+
+SET use_extended_toast_header = off;
+INSERT INTO test_guc_toggle (data)
+ VALUES (repeat('Data created with extended header off. ', 3000));
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 39) AS data_prefix
+FROM test_guc_toggle ORDER BY id;
+ id | compression | data_length | data_prefix
+----+-------------+-------------+-----------------------------------------
+ 1 | pglz | 114000 | Data created with extended header on. D
+ 2 | pglz | 117000 | Data created with extended header off.
+(2 rows)
+
+SET use_extended_toast_header = on;
+SELECT id, length(data) AS data_length FROM test_guc_toggle ORDER BY id;
+ id | data_length
+----+-------------
+ 1 | 114000
+ 2 | 117000
+(2 rows)
+
+-- Cleanup
+DROP SCHEMA test_toast_ext_schema CASCADE;
+DROP EXTENSION test_toast_ext;
diff --git a/src/test/modules/test_toast_ext/expected/test_toast_ext_1.out b/src/test/modules/test_toast_ext/expected/test_toast_ext_1.out
new file mode 100644
index 00000000000..897661fc2a4
--- /dev/null
+++ b/src/test/modules/test_toast_ext/expected/test_toast_ext_1.out
@@ -0,0 +1,37 @@
+--
+-- Tests for extended TOAST header structures and zstd compression
+--
+CREATE EXTENSION test_toast_ext;
+-- Use dedicated schema for test isolation
+CREATE SCHEMA test_toast_ext_schema;
+SET search_path TO test_toast_ext_schema, public;
+-- Compile-time validation tests (always run)
+-- These error out on failure, so completing without error = pass
+SELECT test_toast_structure_sizes();
+ test_toast_structure_sizes
+----------------------------
+
+(1 row)
+
+SELECT test_toast_flag_validation();
+ test_toast_flag_validation
+----------------------------
+
+(1 row)
+
+SELECT test_toast_compression_ids();
+ test_toast_compression_ids
+----------------------------
+
+(1 row)
+
+--
+-- Functional tests for zstd TOAST compression
+-- Skip if not built with USE_ZSTD
+--
+SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE
+ name = 'default_toast_compression' \gset
+\if :skip_test
+ \echo '*** skipping TOAST tests with zstd (not supported) ***'
+*** skipping TOAST tests with zstd (not supported) ***
+ \quit
diff --git a/src/test/modules/test_toast_ext/meson.build b/src/test/modules/test_toast_ext/meson.build
new file mode 100644
index 00000000000..61c07ea1912
--- /dev/null
+++ b/src/test/modules/test_toast_ext/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2022-2025, PostgreSQL Global Development Group
+
+test_toast_ext_sources = files(
+ 'test_toast_ext.c',
+)
+
+if host_system == 'windows'
+ test_toast_ext_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_toast_ext',
+ '--FILEDESC', 'test_toast_ext - test code for extended TOAST headers',])
+endif
+
+test_toast_ext = shared_module('test_toast_ext',
+ test_toast_ext_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_toast_ext
+
+test_install_data += files(
+ 'test_toast_ext.control',
+ 'test_toast_ext--1.0.sql',
+)
+
+tests += {
+ 'name': 'test_toast_ext',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_toast_ext',
+ ],
+ },
+}
diff --git a/src/test/modules/test_toast_ext/sql/test_toast_ext.sql b/src/test/modules/test_toast_ext/sql/test_toast_ext.sql
new file mode 100644
index 00000000000..82e36c57b34
--- /dev/null
+++ b/src/test/modules/test_toast_ext/sql/test_toast_ext.sql
@@ -0,0 +1,136 @@
+--
+-- Tests for extended TOAST header structures and zstd compression
+--
+
+CREATE EXTENSION test_toast_ext;
+
+-- Use dedicated schema for test isolation
+CREATE SCHEMA test_toast_ext_schema;
+SET search_path TO test_toast_ext_schema, public;
+
+-- Compile-time validation tests (always run)
+-- These error out on failure, so completing without error = pass
+SELECT test_toast_structure_sizes();
+SELECT test_toast_flag_validation();
+SELECT test_toast_compression_ids();
+
+--
+-- Functional tests for zstd TOAST compression
+-- Skip if not built with USE_ZSTD
+--
+
+SELECT NOT(enumvals @> '{zstd}') AS skip_test FROM pg_settings WHERE
+ name = 'default_toast_compression' \gset
+\if :skip_test
+ \echo '*** skipping TOAST tests with zstd (not supported) ***'
+ \quit
+\endif
+
+-- Test basic zstd compression
+CREATE TABLE test_zstd_basic (id serial, data text COMPRESSION zstd);
+INSERT INTO test_zstd_basic (data)
+ VALUES (repeat('PostgreSQL zstd TOAST compression test. ', 3000));
+
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 42) AS data_prefix
+FROM test_zstd_basic;
+
+-- Test slice access
+SELECT id, substr(data, 100, 42) AS slice FROM test_zstd_basic;
+
+-- Test UPDATE
+UPDATE test_zstd_basic SET data = repeat('Updated zstd data for TOAST test. ', 3000);
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 35) AS data_prefix
+FROM test_zstd_basic;
+
+-- Test extended header with pglz
+SET use_extended_toast_header = on;
+
+CREATE TABLE test_pglz_extended (data text COMPRESSION pglz);
+INSERT INTO test_pglz_extended (data)
+ VALUES (repeat('PGLZ with extended header format. ', 3000));
+
+SELECT pg_column_compression(data) AS compression,
+ length(data) AS data_length
+FROM test_pglz_extended;
+
+SELECT substr(data, 50, 34) AS slice FROM test_pglz_extended;
+
+-- Test data integrity
+CREATE TABLE test_integrity (
+ method text,
+ original_data text,
+ compressed_data text
+);
+
+INSERT INTO test_integrity VALUES
+ ('pglz', repeat('Integrity test data pattern. ', 2000), NULL),
+ ('zstd', repeat('Integrity test data pattern. ', 2000), NULL);
+
+CREATE TABLE test_pglz_integrity (data text COMPRESSION pglz);
+CREATE TABLE test_zstd_integrity (data text COMPRESSION zstd);
+
+INSERT INTO test_pglz_integrity SELECT original_data FROM test_integrity WHERE method = 'pglz';
+INSERT INTO test_zstd_integrity SELECT original_data FROM test_integrity WHERE method = 'zstd';
+
+SELECT 'pglz' AS method,
+ md5((SELECT original_data FROM test_integrity WHERE method = 'pglz')) =
+ md5((SELECT data FROM test_pglz_integrity)) AS checksum_match;
+
+SELECT 'zstd' AS method,
+ md5((SELECT original_data FROM test_integrity WHERE method = 'zstd')) =
+ md5((SELECT data FROM test_zstd_integrity)) AS checksum_match;
+
+-- Test CLUSTER and VACUUM FULL
+CREATE TABLE test_cluster_zstd (id serial PRIMARY KEY, data text COMPRESSION zstd);
+INSERT INTO test_cluster_zstd (data)
+ VALUES (repeat('Data for CLUSTER test with zstd compression. ', 2500));
+
+SELECT 'before_cluster' AS stage, md5(data) AS hash FROM test_cluster_zstd;
+
+CLUSTER test_cluster_zstd USING test_cluster_zstd_pkey;
+
+SELECT 'after_cluster' AS stage,
+ pg_column_compression(data) AS compression,
+ md5(data) AS hash
+FROM test_cluster_zstd;
+
+VACUUM FULL test_cluster_zstd;
+
+SELECT 'after_vacuum_full' AS stage,
+ pg_column_compression(data) AS compression,
+ md5(data) AS hash
+FROM test_cluster_zstd;
+
+-- Test GUC toggling (mixed formats in same table)
+SET use_extended_toast_header = on;
+CREATE TABLE test_guc_toggle (id serial, data text COMPRESSION pglz);
+INSERT INTO test_guc_toggle (data)
+ VALUES (repeat('Data created with extended header on. ', 3000));
+
+SELECT 'with_ext_on' AS stage,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length
+FROM test_guc_toggle;
+
+SET use_extended_toast_header = off;
+INSERT INTO test_guc_toggle (data)
+ VALUES (repeat('Data created with extended header off. ', 3000));
+
+SELECT id,
+ pg_column_compression(data) AS compression,
+ length(data) AS data_length,
+ left(data, 39) AS data_prefix
+FROM test_guc_toggle ORDER BY id;
+
+SET use_extended_toast_header = on;
+SELECT id, length(data) AS data_length FROM test_guc_toggle ORDER BY id;
+
+-- Cleanup
+DROP SCHEMA test_toast_ext_schema CASCADE;
+DROP EXTENSION test_toast_ext;
diff --git a/src/test/modules/test_toast_ext/test_toast_ext--1.0.sql b/src/test/modules/test_toast_ext/test_toast_ext--1.0.sql
new file mode 100644
index 00000000000..f74d5069fbf
--- /dev/null
+++ b/src/test/modules/test_toast_ext/test_toast_ext--1.0.sql
@@ -0,0 +1,19 @@
+/* src/test/modules/test_toast_ext/test_toast_ext--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_toast_ext" to load this file. \quit
+
+CREATE FUNCTION test_toast_structure_sizes()
+RETURNS void
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION test_toast_flag_validation()
+RETURNS void
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION test_toast_compression_ids()
+RETURNS void
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_toast_ext/test_toast_ext.c b/src/test/modules/test_toast_ext/test_toast_ext.c
new file mode 100644
index 00000000000..59884f2b6d0
--- /dev/null
+++ b/src/test/modules/test_toast_ext/test_toast_ext.c
@@ -0,0 +1,140 @@
+/*-------------------------------------------------------------------------
+ *
+ * test_toast_ext.c
+ * Test module for extended TOAST header structures.
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "access/detoast.h"
+#include "access/toast_compression.h"
+#include "varatt.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(test_toast_structure_sizes);
+PG_FUNCTION_INFO_V1(test_toast_flag_validation);
+PG_FUNCTION_INFO_V1(test_toast_compression_ids);
+
+/*
+ * Verify TOAST structure sizes match expected values.
+ * Errors out if any size is wrong (catches ABI issues).
+ */
+Datum
+test_toast_structure_sizes(PG_FUNCTION_ARGS)
+{
+ /* Standard structure must be 16 bytes */
+ if (sizeof(varatt_external) != 16)
+ elog(ERROR, "varatt_external is %zu bytes, expected 16",
+ sizeof(varatt_external));
+
+ /* Extended structure must be 20 bytes */
+ if (sizeof(varatt_external_extended) != 20)
+ elog(ERROR, "varatt_external_extended is %zu bytes, expected 20",
+ sizeof(varatt_external_extended));
+
+ /* TOAST pointer sizes (include 2-byte external header) */
+ if (TOAST_POINTER_SIZE != 18)
+ elog(ERROR, "TOAST_POINTER_SIZE is %zu, expected 18",
+ (Size) TOAST_POINTER_SIZE);
+
+ if (TOAST_POINTER_SIZE_EXTENDED != 22)
+ elog(ERROR, "TOAST_POINTER_SIZE_EXTENDED is %zu, expected 22",
+ (Size) TOAST_POINTER_SIZE_EXTENDED);
+
+ /* Verify field offsets (no unexpected padding) */
+ if (offsetof(varatt_external_extended, va_rawsize) != 0)
+ elog(ERROR, "va_rawsize offset is %zu, expected 0",
+ offsetof(varatt_external_extended, va_rawsize));
+ if (offsetof(varatt_external_extended, va_extinfo) != 4)
+ elog(ERROR, "va_extinfo offset is %zu, expected 4",
+ offsetof(varatt_external_extended, va_extinfo));
+ if (offsetof(varatt_external_extended, va_flags) != 8)
+ elog(ERROR, "va_flags offset is %zu, expected 8",
+ offsetof(varatt_external_extended, va_flags));
+ if (offsetof(varatt_external_extended, va_data) != 9)
+ elog(ERROR, "va_data offset is %zu, expected 9",
+ offsetof(varatt_external_extended, va_data));
+ if (offsetof(varatt_external_extended, va_valueid) != 12)
+ elog(ERROR, "va_valueid offset is %zu, expected 12",
+ offsetof(varatt_external_extended, va_valueid));
+ if (offsetof(varatt_external_extended, va_toastrelid) != 16)
+ elog(ERROR, "va_toastrelid offset is %zu, expected 16",
+ offsetof(varatt_external_extended, va_toastrelid));
+
+ PG_RETURN_VOID();
+}
+
+/*
+ * Verify flag validation macros work correctly.
+ */
+Datum
+test_toast_flag_validation(PG_FUNCTION_ARGS)
+{
+ /* Valid flags should pass */
+ if (!ExtendedFlagsAreValid(0x00))
+ elog(ERROR, "flags 0x00 should be valid");
+ if (!ExtendedFlagsAreValid(0x01))
+ elog(ERROR, "flags 0x01 should be valid");
+ if (!ExtendedFlagsAreValid(0x02))
+ elog(ERROR, "flags 0x02 should be valid");
+ if (!ExtendedFlagsAreValid(0x03))
+ elog(ERROR, "flags 0x03 should be valid");
+
+ /* Invalid flags should fail */
+ if (ExtendedFlagsAreValid(0x04))
+ elog(ERROR, "flags 0x04 should be invalid");
+ if (ExtendedFlagsAreValid(0x08))
+ elog(ERROR, "flags 0x08 should be invalid");
+ if (ExtendedFlagsAreValid(0xFF))
+ elog(ERROR, "flags 0xFF should be invalid");
+
+ /* Compression methods 0-255 are valid */
+ if (!ExtendedCompressionMethodIsValid(0))
+ elog(ERROR, "compression method 0 should be valid");
+ if (!ExtendedCompressionMethodIsValid(255))
+ elog(ERROR, "compression method 255 should be valid");
+
+ /* Verify method ID constants */
+ if (TOAST_PGLZ_EXT_METHOD != 0)
+ elog(ERROR, "TOAST_PGLZ_EXT_METHOD is %d, expected 0", TOAST_PGLZ_EXT_METHOD);
+ if (TOAST_LZ4_EXT_METHOD != 1)
+ elog(ERROR, "TOAST_LZ4_EXT_METHOD is %d, expected 1", TOAST_LZ4_EXT_METHOD);
+ if (TOAST_ZSTD_EXT_METHOD != 2)
+ elog(ERROR, "TOAST_ZSTD_EXT_METHOD is %d, expected 2", TOAST_ZSTD_EXT_METHOD);
+ if (TOAST_UNCOMPRESSED_EXT_METHOD != 3)
+ elog(ERROR, "TOAST_UNCOMPRESSED_EXT_METHOD is %d, expected 3", TOAST_UNCOMPRESSED_EXT_METHOD);
+
+ PG_RETURN_VOID();
+}
+
+/*
+ * Verify compression ID constants are consistent.
+ */
+Datum
+test_toast_compression_ids(PG_FUNCTION_ARGS)
+{
+ /* Standard compression IDs */
+ if (TOAST_PGLZ_COMPRESSION_ID != 0)
+ elog(ERROR, "TOAST_PGLZ_COMPRESSION_ID is %d, expected 0", TOAST_PGLZ_COMPRESSION_ID);
+ if (TOAST_LZ4_COMPRESSION_ID != 1)
+ elog(ERROR, "TOAST_LZ4_COMPRESSION_ID is %d, expected 1", TOAST_LZ4_COMPRESSION_ID);
+ if (TOAST_INVALID_COMPRESSION_ID != 2)
+ elog(ERROR, "TOAST_INVALID_COMPRESSION_ID is %d, expected 2", TOAST_INVALID_COMPRESSION_ID);
+ if (TOAST_EXTENDED_COMPRESSION_ID != 3)
+ elog(ERROR, "TOAST_EXTENDED_COMPRESSION_ID is %d, expected 3", TOAST_EXTENDED_COMPRESSION_ID);
+
+ /* Extended IDs should match standard where applicable */
+ if (TOAST_PGLZ_EXT_METHOD != TOAST_PGLZ_COMPRESSION_ID)
+ elog(ERROR, "PGLZ IDs mismatch: standard=%d, extended=%d",
+ TOAST_PGLZ_COMPRESSION_ID, TOAST_PGLZ_EXT_METHOD);
+ if (TOAST_LZ4_EXT_METHOD != TOAST_LZ4_COMPRESSION_ID)
+ elog(ERROR, "LZ4 IDs mismatch: standard=%d, extended=%d",
+ TOAST_LZ4_COMPRESSION_ID, TOAST_LZ4_EXT_METHOD);
+
+ PG_RETURN_VOID();
+}
diff --git a/src/test/modules/test_toast_ext/test_toast_ext.control b/src/test/modules/test_toast_ext/test_toast_ext.control
new file mode 100644
index 00000000000..d59ee14ad64
--- /dev/null
+++ b/src/test/modules/test_toast_ext/test_toast_ext.control
@@ -0,0 +1,5 @@
+# test_toast_ext extension
+comment = 'Test module for extended TOAST headers and zstd compression'
+default_version = '1.0'
+module_pathname = '$libdir/test_toast_ext'
+relocatable = true
--
2.39.3 (Apple Git-146)
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected]
Subject: Re: [PATCH] Add zstd compression for TOAST using extended header format
In-Reply-To: <CAOj6k6dkseVvZzmEAWvBd6twZsCU0DbN+qeM7CoDuMM3r9doiw@mail.gmail.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox