postgres.git / summary / log / commit / refs
commit 6e58d6356fcb58f06bfcebb5bfa808660e9531d2
Author: Michael Paquier <michael@paquier.xyz>
Date: Tue Sep 15 06:00:01 2026 +0000
Add support for external on-disk TOAST pointers as oid8
A TOAST table whose chunk_id column is oid8 is now fed a 64-bit object
ID thanks to the introduction of a new vartag_external called
VARTAG_ONDISK_OID8, coupled with a new varatt_external_oid8. The only
difference between the existing varatt_external_oid and the new
varatt_external_oid8 is an extra 4 bytes to store 8 bytes worth an
object ID, spread across two uint32 to force alignement as memcmp may be
used for TOAST pointer comparison (line varatt_external_oid).
There is a one-one mapping between the vartag_external and the type of a
chunk_id:
- An oid8 chunk_id implies the use of VARTAG_ONDISK_OID8.
- An oid chunk_id implies the use of VARTAG_ONDISK_OID.
There are a couple of changes worth noting:
- In toast_tuple_init(), for the comparison of old and new external
values with a memcmp over VARSIZE_EXTERNAL(old_value). Now that
multiple types of external on-disk TOAST pointers are possible, we
need to add a check based on the vartag used by an old or new value.
One of the tests added by c68cba09dd7f cover this case.
- In toast_tuple_find_biggest_attribute(), the initial "biggest_size"
depends on the type of chunk_id. This needs a change in RelationData to
track the chunk_id type; this cached data is filled on demand.
- In detoast.h, toast_external_info_get() and toast_external_data act as
a translation layer to retrieve all the data associated to an Oid or an
Oid8 TOAST pointer, without the need to worry about the vartag dealt
with. The introduction of this function and structure has reduced a lot
of code churn that existed in the initial versions of this patch, as per
a suggestion from Bharath Rupireddy.
- amcheck is updated to check the vartag <-> chunk_id type mapping
requirements, handling both vartags depending on the underlying type of
chunk_id on the TOAST table.
Bump catalog version, due to at least the change of VARTAG_SIZE().
(This counts for the bump forgotten in a625fc570c22..)
Author: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Greg Burd <greg@burd.me>
Reviewed-by: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Reviewed-by: Yugo Nagata <nagata@sraoss.co.jp>
Discussion: https://postgr.es/m/af19kUjwjhaoUTLn@paquier.xyz
contrib/amcheck/verify_heapam.c | 85 ++++++---
doc/src/sgml/storage.sgml | 12 +-
src/backend/access/common/detoast.c | 71 ++++---
src/backend/access/common/toast_compression.c | 8 +-
src/backend/access/common/toast_internals.c | 238 ++++++++++++++----------
src/backend/access/heap/heaptoast.c | 2 +-
src/backend/access/table/toast_helper.c | 22 ++-
src/backend/replication/logical/reorderbuffer.c | 14 +-
src/backend/utils/adt/varlena.c | 6 +-
src/backend/utils/cache/relcache.c | 37 ++++
src/include/access/detoast.h | 48 +++++
src/include/access/heaptoast.h | 22 ++-
src/include/catalog/catversion.h | 2 +-
src/include/utils/rel.h | 9 +
src/include/utils/relcache.h | 1 +
src/include/varatt.h | 78 +++++---
src/tools/pgindent/typedefs.list | 2 +
17 files changed, 448 insertions(+), 209 deletions(-)
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 56167ecfe1d..8c57adc5d54 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -74,7 +74,9 @@ typedef enum SkipPages
*/
typedef struct ToastedAttribute
{
- varatt_external_oid toast_pointer;
+ Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */
+ uint32 va_extinfo; /* external size and compression method */
+ vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */
BlockNumber blkno; /* block in main table */
OffsetNumber offnum; /* offset in main table */
AttrNumber attnum; /* attribute in main table */
@@ -189,7 +191,7 @@ static void check_tuple(HeapCheckContext *ctx,
XidCommitStatus *xmin_commit_status);
static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
ToastedAttribute *ta, int32 *expected_chunk_seq,
- uint32 extsize);
+ uint32 extsize, int32 max_chunk_size);
static bool check_tuple_attribute(HeapCheckContext *ctx);
static void check_toasted_attribute(HeapCheckContext *ctx,
@@ -1554,7 +1556,7 @@ check_tuple_visibility(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
static void
check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
ToastedAttribute *ta, int32 *expected_chunk_seq,
- uint32 extsize)
+ uint32 extsize, int32 max_chunk_size)
{
int32 chunk_seq;
int32 last_chunk_seq;
@@ -1562,12 +1564,9 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
bool isnull;
int32 chunksize;
int32 expected_size;
- int32 max_chunk_size;
- Oid8 toast_valueid;
+ Oid8 toast_valueid = ta->va_valueid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
last_chunk_seq = (extsize - 1) / max_chunk_size;
- toast_valueid = ta->toast_pointer.va_valueid;
/* Sanity-check the sequence number. */
chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2,
@@ -1671,8 +1670,11 @@ check_tuple_attribute(HeapCheckContext *ctx)
char *tp; /* pointer to the tuple data */
uint16 infomask;
Oid8 toast_pointer_valueid;
+ int32 va_rawsize;
+ uint32 va_extinfo;
CompactAttribute *thisatt;
- varatt_external_oid toast_pointer;
+ vartag_external va_tag_value;
+ toast_external_data toast_ext_data;
infomask = ctx->tuphdr->t_infomask;
thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
@@ -1731,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
{
uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
- if (va_tag != VARTAG_ONDISK_OID)
+ if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8)
{
report_corruption(ctx,
psprintf("toasted attribute has unexpected TOAST tag %u",
@@ -1775,27 +1777,28 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* It is external, and we're looking at a page on disk */
- /*
- * Must copy attr into toast_pointer for alignment considerations
- */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- toast_pointer_valueid = toast_pointer.va_valueid;
+ /* Must copy attr into a decoded pointer for alignment considerations */
+ toast_external_info_get(attr, &toast_ext_data);
+ va_tag_value = toast_ext_data.tag;
+ toast_pointer_valueid = toast_ext_data.valueid;
+ va_rawsize = toast_ext_data.rawsize;
+ va_extinfo = toast_ext_data.extinfo;
/* Toasted attributes too large to be untoasted should never be stored */
- if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
+ if (va_rawsize > VARLENA_SIZE_LIMIT)
report_corruption(ctx,
psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d",
toast_pointer_valueid,
- toast_pointer.va_rawsize,
+ va_rawsize,
VARLENA_SIZE_LIMIT));
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
{
ToastCompressionId cmid;
bool valid = false;
/* Compressed attributes should have a valid compression method */
- cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+ cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
switch (cmid)
{
/* List of all valid compression method IDs */
@@ -1849,7 +1852,10 @@ check_tuple_attribute(HeapCheckContext *ctx)
ta = palloc0_object(ToastedAttribute);
- VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+ /* The pointer has already been decoded above, just reuse it */
+ ta->tag = va_tag_value;
+ ta->va_valueid = toast_pointer_valueid;
+ ta->va_extinfo = va_extinfo;
ta->blkno = ctx->blkno;
ta->offnum = ctx->offnum;
ta->attnum = ctx->attnum;
@@ -1878,18 +1884,44 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
int32 max_chunk_size;
Oid8 toast_valueid;
Oid toast_typid;
+ vartag_external expected_tag = VARTAG_ONDISK_OID;
+
+ toast_valueid = ta->va_valueid;
+ extsize = VARATT_EXTINFO_GET_EXTSIZE(ta->va_extinfo);
+ /*
+ * Take the chunk_id type from the TOAST table's own definition, not from
+ * from the vartag in the main table as that pointer is the very thing
+ * under scrutiny here. The two must agree.
+ */
toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ if (toast_typid == OID8OID)
+ expected_tag = VARTAG_ONDISK_OID8;
+ else if (toast_typid == OIDOID)
+ expected_tag = VARTAG_ONDISK_OID;
+ else
+ {
+ report_toast_corruption(ctx, ta,
+ psprintf("toast value " OID8_FORMAT " stored in toast table whose chunk_id has unexpected type %u",
+ toast_valueid, toast_typid));
+ }
+
+ if (ta->tag != expected_tag)
+ {
+ report_toast_corruption(ctx, ta,
+ psprintf("toast value " OID8_FORMAT " has TOAST tag %u, but chunk_id of toast table has type %u",
+ toast_valueid, (uint8) ta->tag,
+ toast_typid));
+ return;
+ }
- extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer);
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
last_chunk_seq = (extsize - 1) / max_chunk_size;
/*
- * Setup a scan key to find chunks in toast table with matching va_valueid
+ * Setup a scan key to find chunks in toast table with matching value ID
*/
- toast_valueid_scankey_init(&toastkey, toast_typid,
- ta->toast_pointer.va_valueid);
+ toast_valueid_scankey_init(&toastkey, toast_typid, toast_valueid);
/*
* Check if any chunks for this toasted object exist in the toast table,
@@ -1905,12 +1937,11 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
ForwardScanDirection)) != NULL)
{
found_toasttup = true;
- check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize);
+ check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize,
+ max_chunk_size);
}
systable_endscan_ordered(toastscan);
- toast_valueid = ta->toast_pointer.va_valueid;
-
if (!found_toasttup)
report_toast_corruption(ctx, ta,
psprintf("toast value " OID8_FORMAT " not found in toast table",
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 21bd67c11ea..83de016eaa5 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -417,8 +417,10 @@ described in more detail below.
<para>
Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> bytes (by default this value is chosen
-so that four chunk rows will fit on a page, making it about 2000 bytes).
+most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> or
+<symbol>TOAST_OID8_MAX_CHUNK_SIZE</symbol> bytes depending on the
+<structfield>chunk_id</structfield> type (by default these values are chosen
+so that four chunk rows will fit on a page, making them about 2000 bytes).
Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
belonging to the owning table. Every
<acronym>TOAST</acronym> table has the columns
@@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk
logical datum size (original uncompressed data length), physical stored size
(different if compression was applied), and the compression method used, if
any. Allowing for the varlena header bytes,
-the total size of an on-disk <acronym>TOAST</acronym> pointer datum is therefore 18
-bytes regardless of the actual size of the represented value.
+the total size of an on-disk <acronym>TOAST</acronym> pointer datum is 18
+bytes when using an OID as <structfield>chunk_id</structfield>, or 22 bytes
+when using an OID8 as <structfield>chunk_id</structfield>, regardless of the
+actual size of the represented value.
</para>
<para>
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index a50aea7b6b1..06992de1ede 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -225,12 +225,18 @@ detoast_attr_slice(varlena *attr,
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
+ int32 extsize;
+ uint32 compress_method;
+ bool is_compressed;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
+ extsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ compress_method = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
+ is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize);
/* fast path for non-compressed external datums */
- if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (!is_compressed)
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/*
@@ -240,7 +246,7 @@ detoast_attr_slice(varlena *attr,
*/
if (slicelimit >= 0)
{
- int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ int32 max_size = extsize;
/*
* Determine maximum amount of compressed data needed for a prefix
@@ -251,8 +257,7 @@ detoast_attr_slice(varlena *attr,
* determine how much compressed data we need to be sure of being
* able to decompress the required slice.
*/
- if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) ==
- TOAST_PGLZ_COMPRESSION_ID)
+ if (compress_method == TOAST_PGLZ_COMPRESSION_ID)
max_size = pglz_maximum_compressed_size(slicelimit, max_size);
/*
@@ -344,20 +349,22 @@ toast_fetch_datum(varlena *attr)
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
-
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toast_external_info_get(attr, &toast_ext_data);
+ attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ toastrelid = toast_ext_data.toastrelid;
+ valueid = toast_ext_data.valueid;
result = (varlena *) palloc(attrsize + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
else
SET_VARSIZE(result, attrsize + VARHDRSZ);
@@ -369,10 +376,10 @@ toast_fetch_datum(varlena *attr)
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, 0, attrsize, result);
/* Close toast table */
@@ -398,23 +405,27 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
+ bool is_compressed;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
+ attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ toastrelid = toast_ext_data.toastrelid;
+ valueid = toast_ext_data.valueid;
+ is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize);
/*
* It's nonsense to fetch slices of a compressed datum unless when it's a
* prefix -- this isn't lo_* we can't return a compressed datum which is
* meaningful to toast later.
*/
- Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
-
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ Assert(!is_compressed || 0 == sliceoffset);
if (sliceoffset >= attrsize)
{
@@ -427,7 +438,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
* space required by va_tcinfo, which is stored at the beginning as an
* int32 value.
*/
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0)
+ if (is_compressed && slicelength > 0)
slicelength = slicelength + sizeof(int32);
/*
@@ -440,7 +451,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
result = (varlena *) palloc(slicelength + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
else
SET_VARSIZE(result, slicelength + VARHDRSZ);
@@ -449,10 +460,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
return result; /* Can save a lot of work at this point! */
/* Open the toast relation */
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, sliceoffset, slicelength,
result);
@@ -550,10 +561,10 @@ toast_raw_datum_size(Datum value)
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/* va_rawsize is the size of the original datum -- including header */
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_rawsize;
+ toast_external_info_get(attr, &toast_ext_data);
+ result = toast_ext_data.rawsize;
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
@@ -610,10 +621,10 @@ toast_datum_size(Datum value)
* compressed or not. We do not count the size of the toast pointer
* ... should we?
*/
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toast_external_info_get(attr, &toast_ext_data);
+ result = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 3b1bd651926..09cdf880e4f 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -262,12 +262,12 @@ toast_get_compression_id(varlena *attr)
*/
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
- cmid = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer);
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
+ cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
}
else if (VARATT_IS_COMPRESSED(attr))
cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 2b7c6433bb4..a4e8a096254 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -25,7 +25,6 @@
#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
-#include "utils/lsyscache.h"
static bool toastrel_valueid_exists(Relation toastrel, Oid8 valueid);
static bool toastid_valueid_exists(Oid toastrelid, Oid8 valueid);
@@ -104,6 +103,56 @@ toast_compress_datum(Datum value, char cmethod)
}
}
+/* ----------
+ * toast_preserve_valueid -
+ *
+ * During a table rewrite that preserves rd_toastoid, we want to preserve
+ * toast value IDs too. If the datum previously had an external value from
+ * that same toast table, return its value ID so the caller can re-use it,
+ * otherwise return InvalidOid8.
+ *
+ * This works for both Oid and Oid8 value IDs, as the value ID is decoded
+ * independently of the pointer's vartag.
+ *
+ * There is a corner case here: the table rewrite might have to copy both
+ * live and recently-dead versions of a row, and those versions could easily
+ * reference the same toast value. When we copy the second or later version
+ * of such a row, preserving the value ID means we select one that's already
+ * in the new toast table. We detect that and set *data_todo to 0 so the
+ * caller falls through without writing the data again.
+ *
+ * While annoying and ugly-looking, this is a good thing because it ensures
+ * that we wind up with only one copy of the toast value when there is only
+ * one copy in the old toast table. Before we detected this case, we'd have
+ * made multiple copies, wasting space; and what's worse, the copies
+ * belonging to already-deleted heap tuples would not be reclaimed by VACUUM.
+ * ----------
+ */
+static Oid8
+toast_preserve_valueid(Relation toastrel, Oid toastoid,
+ varlena *oldexternal, int32 *data_todo)
+{
+ toast_external_data old;
+
+ if (!OidIsValid(toastoid) || oldexternal == NULL)
+ return InvalidOid8;
+
+ Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
+ toast_external_info_get(oldexternal, &old);
+
+ /*
+ * Only re-use the value ID if the old pointer came from this same toast
+ * table.
+ */
+ if (old.toastrelid != toastoid)
+ return InvalidOid8;
+
+ if (toastrel_valueid_exists(toastrel, old.valueid))
+ *data_todo = 0;
+
+ return old.valueid;
+}
+
/* ----------
* toast_save_datum -
*
@@ -125,14 +174,20 @@ toast_save_datum(Relation rel, Datum value,
TupleDesc toasttupDesc;
CommandId mycid = GetCurrentCommandId(true);
varlena *result;
- varatt_external_oid toast_pointer;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
int num_indexes;
int validIndex;
- Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1);
+ Oid toast_typid = RelationGetToastChunkIdType(rel);
+ int32 max_chunk_size;
+
+ /* Fields that will be assembled into the TOAST pointer at the end */
+ int32 va_rawsize;
+ uint32 va_extinfo;
+ Oid8 va_valueid;
+ Oid va_toastrelid;
Assert(!VARATT_IS_EXTERNAL(dval));
@@ -164,28 +219,32 @@ toast_save_datum(Relation rel, Datum value,
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
- toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = data_todo + VARHDRSZ; /* as if not short */
+ va_extinfo = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
+ uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
- toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
+ va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
/* set external size and compression method */
- VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
- VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+ Assert(cmid == TOAST_PGLZ_COMPRESSION_ID ||
+ cmid == TOAST_LZ4_COMPRESSION_ID);
+ va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS);
+
/* Assert that the numbers look like it's compressed */
- Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer));
+ Assert(VARATT_EXTINFO_IS_COMPRESSED(va_extinfo, va_rawsize));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
- toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = VARSIZE(dval);
+ va_extinfo = data_todo;
}
/*
@@ -197,100 +256,44 @@ toast_save_datum(Relation rel, Datum value,
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
- toast_pointer.va_toastrelid = rel->rd_toastoid;
+ va_toastrelid = rel->rd_toastoid;
else
- toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
+ va_toastrelid = RelationGetRelid(toastrel);
/*
- * Choose a new value to use as the value ID for this toast value, be it
- * for OID or OID8 TOAST relations.
- *
- * Normally we just choose an unused value within the toast table. But
- * during table-rewriting operations where we are preserving an existing
- * toast table OID, we want to preserve toast value IDs too. So, if
- * rd_toastoid is set and we had a prior external value from that same
- * toast table, re-use its value ID. If we didn't have a prior external
- * value (which is a corner case, but possible if the table's attstorage
- * options have been changed), we have to pick a value ID that doesn't
- * conflict with either new or existing toast value IDs. If the TOAST
- * table uses 8-byte value IDs, we should not really care much about that.
+ * Choose the value ID for this toast value. During a table rewrite that
+ * preserves rd_toastoid we re-use the prior value ID if we can (see
+ * toast_preserve_valueid); otherwise we pick a fresh one. For Oid, it
+ * must not conflict with old or new toast values, while for Oid8 the ID
+ * space is large enough that conflicts are not a concern.
*/
- if (!OidIsValid(rel->rd_toastoid))
+ va_valueid = toast_preserve_valueid(toastrel, rel->rd_toastoid,
+ oldexternal, &data_todo);
+ if (va_valueid == InvalidOid8)
{
- /* normal case: just choose an unused OID */
if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
- else
{
- toast_pointer.va_valueid =
- GetNewOidWithIndex(toastrel,
- RelationGetRelid(toastidxs[validIndex]),
- (AttrNumber) 1);
- }
- }
- else
- {
- /* rewrite case: check to see if value was in old toast table */
- toast_pointer.va_valueid = InvalidOid;
- if (oldexternal != NULL)
- {
- varatt_external_oid old_toast_pointer;
-
- Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
- if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
- {
- /* This value came from the old toast table; reuse its OID */
- toast_pointer.va_valueid = old_toast_pointer.va_valueid;
-
- /*
- * There is a corner case here: the table rewrite might have
- * to copy both live and recently-dead versions of a row, and
- * those versions could easily reference the same toast value.
- * When we copy the second or later version of such a row,
- * reusing the OID will mean we select an OID that's already
- * in the new toast table. Check for that, and if so, just
- * fall through without writing the data again.
- *
- * While annoying and ugly-looking, this is a good thing
- * because it ensures that we wind up with only one copy of
- * the toast value when there is only one copy in the old
- * toast table. Before we detected this case, we'd have made
- * multiple copies, wasting space; and what's worse, the
- * copies belonging to already-deleted heap tuples would not
- * be reclaimed by VACUUM.
- */
- if (toastrel_valueid_exists(toastrel,
- toast_pointer.va_valueid))
- {
- /* Match, so short-circuit the data storage loop below */
- data_todo = 0;
- }
- }
+ /* The Oid8 space is large enough that we need not check conflicts */
+ va_valueid = GetNewObjectId8();
}
- if (toast_pointer.va_valueid == InvalidOid)
+ else
{
/*
- * new value; must choose a value that doesn't conflict in either
- * old or new toast table.
+ * Choose an unused OID. During a rewrite we must also avoid IDs
+ * that are still present in the old toast table.
*/
- if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
- else
- {
- do
- {
- toast_pointer.va_valueid =
- GetNewOidWithIndex(toastrel,
- RelationGetRelid(toastidxs[validIndex]),
- (AttrNumber) 1);
- } while (toastid_valueid_exists(rel->rd_toastoid,
- toast_pointer.va_valueid));
- }
+ do
+ va_valueid =
+ GetNewOidWithIndex(toastrel,
+ RelationGetRelid(toastidxs[validIndex]),
+ (AttrNumber) 1);
+ while (OidIsValid(rel->rd_toastoid) &&
+ toastid_valueid_exists(rel->rd_toastoid, va_valueid));
}
}
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
+
/*
* Split up the item into chunks
*/
@@ -302,8 +305,10 @@ toast_save_datum(Relation rel, Datum value,
union
{
alignas(int32) varlena hdr;
+
/* this is to make the union big enough for a chunk: */
- char data[TOAST_OID_MAX_CHUNK_SIZE + VARHDRSZ];
+ char data[Max(TOAST_OID_MAX_CHUNK_SIZE,
+ TOAST_OID8_MAX_CHUNK_SIZE) + VARHDRSZ];
} chunk_data;
int32 chunk_size;
@@ -312,15 +317,15 @@ toast_save_datum(Relation rel, Datum value,
/*
* Calculate the size of this chunk
*/
- chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo);
+ chunk_size = Min(max_chunk_size, data_todo);
/*
* Build a tuple and store it
*/
if (toast_typid == OID8OID)
- t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid);
+ t_values[0] = ObjectId8GetDatum(va_valueid);
else
- t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
+ t_values[0] = ObjectIdGetDatum((Oid) va_valueid);
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
@@ -376,9 +381,32 @@ toast_save_datum(Relation rel, Datum value,
/*
* Create the TOAST pointer value that we'll return
*/
- result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
- SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
- memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ if (toast_typid == OID8OID)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ VARATT_EXTERNAL_OID8_SET_VALUEID(&toast_pointer, va_valueid);
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ toast_pointer.va_valueid = (Oid) va_valueid;
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
return PointerGetDatum(result);
}
@@ -413,7 +441,7 @@ void
toast_delete_datum(Relation rel, Datum value, bool is_speculative)
{
varlena *attr = (varlena *) DatumGetPointer(value);
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
Relation toastrel;
Relation *toastidxs;
ScanKeyData toastkey;
@@ -425,13 +453,17 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
return;
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Decode the pointer to get the toast relation OID and value ID. The
+ * vartag tells us everything we need - no TOAST table schema lookup
+ * required.
+ */
+ toast_external_info_get(attr, &toast_ext_data);
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
+ toastrel = table_open(toast_ext_data.toastrelid, RowExclusiveLock);
/* Fetch valid relation used for process */
validIndex = toast_open_indexes(toastrel,
@@ -444,7 +476,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
*/
toast_valueid_scankey_init(&toastkey,
TupleDescAttr(toastrel->rd_att, 0)->atttypid,
- toast_pointer.va_valueid);
+ toast_ext_data.valueid);
/*
* Find all the chunks. (We don't actually care whether we see them in
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index 81d5085ca39..f8a54ff2582 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -649,7 +649,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize,
&num_indexes);
toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
totalchunks = ((attrsize - 1) / max_chunk_size) + 1;
startchunk = sliceoffset / max_chunk_size;
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index 6f2d691cc68..2613d9dc095 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -69,12 +69,16 @@ toast_tuple_init(ToastTupleContext *ttc)
/*
* If the old value is stored on disk, check if it has changed so
* we have to delete it later.
+ *
+ * Note that TOAST pointers could have different vartags, for oid
+ * or oid8, and these can have different sizes.
*/
if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
VARATT_IS_EXTERNAL_ONDISK(old_value))
{
if (ttc->ttc_isnull[i] ||
!VARATT_IS_EXTERNAL_ONDISK(new_value) ||
+ VARTAG_EXTERNAL(old_value) != VARTAG_EXTERNAL(new_value) ||
memcmp(old_value, new_value,
VARSIZE_EXTERNAL(old_value)) != 0)
{
@@ -171,8 +175,9 @@ toast_tuple_init(ToastTupleContext *ttc)
* The column must have attstorage EXTERNAL or EXTENDED if check_main is
* false, and must have attstorage MAIN if check_main is true.
*
- * The column must have a minimum size of MAXALIGN(TOAST_OID_POINTER_SIZE);
- * if not, no benefit is to be expected by compressing it.
+ * The column must be larger than the TOAST pointer that would replace it;
+ * if not, no benefit is to be expected by compressing it. Note that this
+ * choice depends on the TOAST value type, oid or oid8.
*
* The return value is the index of the biggest suitable column, or
* -1 if there is none.
@@ -184,10 +189,21 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
int numAttrs = tupleDesc->natts;
int biggest_attno = -1;
- int32 biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE);
+ int32 biggest_size;
int32 skip_colflags = TOASTCOL_IGNORE;
int i;
+ /*
+ * Size of the TOAST pointer this relation would use. A relation without
+ * a TOAST table cannot have any of its attributes moved out-of-line, but
+ * it can still have some of them compressed. Fall back to the oid size
+ * in that case.
+ */
+ if (RelationGetToastChunkIdType(ttc->ttc_rel) == OID8OID)
+ biggest_size = MAXALIGN(TOAST_OID8_POINTER_SIZE);
+ else
+ biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE);
+
if (for_compression)
skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index dd4b6e8f7f4..ede117d339e 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -5175,7 +5175,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
varlena *varlena_pointer;
/* va_rawsize is the size of the original datum -- including header */
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
varatt_indirect redirect_pointer;
varlena *new_datum = NULL;
varlena *reconstructed;
@@ -5198,11 +5198,11 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
varlena_pointer = (varlena *) DatumGetPointer(attrs[natt]);
/* no need to do anything if the tuple isn't external */
- if (!VARATT_IS_EXTERNAL(varlena_pointer))
+ if (!VARATT_IS_EXTERNAL_ONDISK(varlena_pointer))
continue;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer);
- toast_valueid = toast_pointer.va_valueid;
+ toast_external_info_get(varlena_pointer, &toast_ext_data);
+ toast_valueid = toast_ext_data.valueid;
/*
* Check whether the toast tuple changed, replace if so.
@@ -5220,7 +5220,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
free[natt] = true;
- reconstructed = palloc0(toast_pointer.va_rawsize);
+ reconstructed = palloc0(toast_ext_data.rawsize);
ent->reconstructed = reconstructed;
@@ -5245,10 +5245,10 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer));
+ Assert(data_done == VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo));
/* make sure its marked as compressed or not */
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
else
SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 4533dac11a6..01cc61c5778 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -4261,7 +4261,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
{
int typlen;
varlena *attr;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
/* On first call, get the input type's typlen, and save at *fn_extra */
if (fcinfo->flinfo->fn_extra == NULL)
@@ -4288,9 +4288,9 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
PG_RETURN_NULL();
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
- PG_RETURN_OID8(toast_pointer.va_valueid);
+ PG_RETURN_OID8(toast_ext_data.valueid);
}
/*
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 541f30f0972..2555de93706 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -1205,6 +1205,9 @@ retry:
relation->rd_fkeylist = NIL;
relation->rd_fkeyvalid = false;
+ /* TOAST type data is not loaded till asked for */
+ relation->rd_toastchunkidtype = InvalidOid;
+
/* partitioning data is not loaded till asked for */
relation->rd_partkey = NULL;
relation->rd_partkeycxt = NULL;
@@ -5100,6 +5103,39 @@ RelationGetReplicaIndex(Relation relation)
return relation->rd_replidindex;
}
+/*
+ * RelationGetToastChunkIdType -- get the type of the relation's TOAST
+ * table "chunk_id" column
+ *
+ * Returns OIDOID or OID8OID, or InvalidOid if the relation has no TOAST
+ * table.
+ */
+Oid
+RelationGetToastChunkIdType(Relation relation)
+{
+ Oid toastrelid = relation->rd_rel->reltoastrelid;
+ Oid typid;
+
+ /* Quick exit if we already computed the value */
+ if (OidIsValid(relation->rd_toastchunkidtype))
+ return relation->rd_toastchunkidtype;
+
+ /* Nothing to report without a TOAST table */
+ if (!OidIsValid(toastrelid))
+ return InvalidOid;
+
+ typid = get_atttype(toastrelid, 1);
+ if (!OidIsValid(typid))
+ elog(ERROR, "cache lookup failed for TOAST relation %u",
+ toastrelid);
+ if (typid != OIDOID && typid != OID8OID)
+ elog(ERROR, "unexpected type %u for chunk_id in TOAST relation %u",
+ typid, toastrelid);
+
+ relation->rd_toastchunkidtype = typid;
+ return typid;
+}
+
/*
* RelationGetIndexExpressions -- get the index expressions for an index
*
@@ -6545,6 +6581,7 @@ load_relcache_init_file(bool shared)
rel->rd_firstRelfilelocatorSubid = InvalidSubTransactionId;
rel->rd_droppedSubid = InvalidSubTransactionId;
rel->rd_amcache = NULL;
+ rel->rd_toastchunkidtype = InvalidOid;
rel->pgstat_info = NULL;
/*
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index 2b1428b1e81..93b7a253760 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -12,6 +12,8 @@
#ifndef DETOAST_H
#define DETOAST_H
+#include "varatt.h"
+
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
* into a local "varatt_external_oid" toast pointer. This should be
@@ -30,9 +32,55 @@ do { \
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
#define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid))
+/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */
+#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8))
+
/* Size of an EXTERNAL datum that contains an indirection pointer */
#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
+/*
+ * Decoded contents of an on-disk external TOAST pointer.
+ */
+typedef struct toast_external_data
+{
+ vartag_external tag; /* VARTAG_ONDISK_* */
+ int32 rawsize; /* original data size (includes header) */
+ uint32 extinfo; /* saved size + compression method */
+ Oid8 valueid; /* value ID (can be widened from Oid) */
+ Oid toastrelid; /* OID of the TOAST table containing it */
+} toast_external_data;
+
+/*
+ * Decode an on-disk external TOAST pointer into a toast_external_data.
+ */
+static inline void
+toast_external_info_get(const struct varlena *attr, toast_external_data *toast_ext_data)
+{
+ Assert(VARATT_IS_EXTERNAL_ONDISK(attr));
+
+ toast_ext_data->tag = VARTAG_EXTERNAL(attr);
+ if (toast_ext_data->tag == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_ext_data->rawsize = toast_pointer.va_rawsize;
+ toast_ext_data->extinfo = toast_pointer.va_extinfo;
+ toast_ext_data->valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer);
+ toast_ext_data->toastrelid = toast_pointer.va_toastrelid;
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_ext_data->rawsize = toast_pointer.va_rawsize;
+ toast_ext_data->extinfo = toast_pointer.va_extinfo;
+ toast_ext_data->valueid = toast_pointer.va_valueid;
+ toast_ext_data->toastrelid = toast_pointer.va_toastrelid;
+ }
+}
+
/* ----------
* detoast_external_attr() -
*
diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index aaae6e0fef6..4c3fda8a5f9 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -69,13 +69,14 @@
/*
* When we store an oversize datum externally, we divide it into chunks
- * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must*
- * be small enough that the completed toast-table tuple (including the
- * ID and sequence fields and all overhead) will fit on a page.
+ * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE
+ * data bytes, depending on the chunk_id type of the TOAST table. These
+ * numbers *must* be small enough that the completed toast-table tuple
+ * (including the ID and sequence fields and all overhead) will fit on a page.
* The coding here sets the size on the theory that we want to fit
* EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
*
- * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb.
+ * NB: Changing these values requires an initdb.
*/
#define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
@@ -88,6 +89,19 @@
sizeof(int32) - \
VARHDRSZ)
+#define TOAST_OID8_MAX_CHUNK_SIZE \
+ (EXTERN_TUPLE_MAX_SIZE - \
+ MAXALIGN(SizeofHeapTupleHeader) - \
+ sizeof(Oid8) - \
+ sizeof(int32) - \
+ VARHDRSZ)
+
+/*
+ * Select the chunk size for a TOAST table from its value type.
+ */
+#define TOAST_MAX_CHUNK_SIZE(toast_typid) \
+ ((toast_typid) == OID8OID ? TOAST_OID8_MAX_CHUNK_SIZE : TOAST_OID_MAX_CHUNK_SIZE)
+
/* ----------
* heap_toast_insert_or_update -
*
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index f28b8db6e6f..c192c0a5a00 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 202609141
+#define CATALOG_VERSION_NO 202609151
#endif
diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h
index d61432de705..9c8d17337b8 100644
--- a/src/include/utils/rel.h
+++ b/src/include/utils/rel.h
@@ -250,6 +250,15 @@ typedef struct RelationData
*/
Oid rd_toastoid; /* Real TOAST table's OID, or InvalidOid */
+ /*
+ * Type OID of the "chunk_id" column of this relation's TOAST table, i.e.
+ * OIDOID or OID8OID.
+ *
+ * This data is filled on demand by RelationGetToastChunkIdType(), not at
+ * relcache build time, so as to save in syscache lookups.
+ */
+ Oid rd_toastchunkidtype;
+
bool pgstat_enabled; /* should relation stats be counted */
/* use "struct" here to avoid needing to include pgstat.h: */
struct PgStat_RelationStatus *pgstat_info; /* statistics collection area */
diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h
index 89c27aa1529..e17c8490833 100644
--- a/src/include/utils/relcache.h
+++ b/src/include/utils/relcache.h
@@ -57,6 +57,7 @@ extern List *RelationGetIndexList(Relation relation);
extern List *RelationGetStatExtList(Relation relation);
extern Oid RelationGetPrimaryKeyIndex(Relation relation, bool deferrable_ok);
extern Oid RelationGetReplicaIndex(Relation relation);
+extern Oid RelationGetToastChunkIdType(Relation relation);
extern List *RelationGetIndexExpressions(Relation relation);
extern List *RelationGetDummyIndexExpressions(Relation relation);
extern List *RelationGetIndexPredicate(Relation relation);
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 89b47db0833..77041f7673a 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -42,6 +42,44 @@ StaticAssertDecl((sizeof(int32) + sizeof(uint32) + 2 * sizeof(Oid)) ==
sizeof(varatt_external_oid),
"varatt_external_oid should have no padding");
+/*
+ * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use
+ * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid
+ * except for the value ID, which is 8 bytes wide. The value ID is split
+ * into two uint32 to force alignment.
+ *
+ * This struct must not contain any padding, because we sometimes compare
+ * these pointers using memcmp.
+ */
+typedef struct varatt_external_oid8
+{
+ int32 va_rawsize; /* Original data size (includes header) */
+ uint32 va_extinfo; /* External saved size (without header) and
+ * compression method */
+ uint32 va_valueid_lo; /* Low 32 bits of value ID */
+ uint32 va_valueid_hi; /* High 32 bits of value ID */
+ Oid va_toastrelid; /* RelID of TOAST table containing it */
+} varatt_external_oid8;
+
+StaticAssertDecl((sizeof(int32) + 3 * sizeof(uint32) + sizeof(Oid)) ==
+ sizeof(varatt_external_oid8),
+ "varatt_external_oid8 should have no padding");
+
+/* Get/set the 64-bit value ID from the lo/hi halves */
+static inline Oid8
+VARATT_EXTERNAL_OID8_GET_VALUEID(varatt_external_oid8 toast_pointer)
+{
+ return ((Oid8) toast_pointer.va_valueid_lo) |
+ (((Oid8) toast_pointer.va_valueid_hi) << 32);
+}
+
+static inline void
+VARATT_EXTERNAL_OID8_SET_VALUEID(varatt_external_oid8 *toast_pointer, Oid8 id)
+{
+ toast_pointer->va_valueid_lo = (uint32) id;
+ toast_pointer->va_valueid_hi = (uint32) (id >> 32);
+}
+
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
@@ -91,6 +129,7 @@ typedef enum vartag_external
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
+ VARTAG_ONDISK_OID8 = 4,
VARTAG_ONDISK_OID = 18
} vartag_external;
@@ -112,6 +151,8 @@ VARTAG_SIZE(vartag_external tag)
return sizeof(varatt_expanded);
else if (tag == VARTAG_ONDISK_OID)
return sizeof(varatt_external_oid);
+ else if (tag == VARTAG_ONDISK_OID8)
+ return sizeof(varatt_external_oid8);
else
{
Assert(false);
@@ -365,7 +406,12 @@ VARATT_IS_EXTERNAL(const void *PTR)
static inline bool
VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
{
- return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID;
+ vartag_external tag;
+
+ if (!VARATT_IS_EXTERNAL(PTR))
+ return false;
+ tag = VARTAG_EXTERNAL(PTR);
+ return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8);
}
/* Is varlena datum an indirect pointer? */
@@ -508,43 +554,31 @@ VARDATA_COMPRESSED_GET_COMPRESS_METHOD(const void *PTR)
}
/*
- * Same for external Datums; but note argument is a struct
- * varatt_external_oid.
+ * Same for external Datums, saved into an va_extinfo.
*/
static inline Size
-VARATT_EXTERNAL_OID_GET_EXTSIZE(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_GET_EXTSIZE(uint32 extinfo)
{
- return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
+ return extinfo & VARLENA_EXTSIZE_MASK;
}
static inline uint32
-VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_GET_COMPRESS_METHOD(uint32 extinfo)
{
- return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
+ return extinfo >> VARLENA_EXTSIZE_BITS;
}
-/* Set size and compress method of an externally-stored varlena datum */
-/* This has to remain a macro; beware multiple evaluations! */
-#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
- do { \
- Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
- } while (0)
-
/*
- * Testing whether an externally-stored value is compressed now requires
- * comparing size stored in va_extinfo (the actual length of the external data)
+ * Testing whether an externally-stored value is compressed requires comparing
+ * the saved size stored in va_extinfo (the actual length of the external data)
* to rawsize (the original uncompressed datum's size). The latter includes
* VARHDRSZ overhead, the former doesn't. We never use compression unless it
* actually saves space, so we expect either equality or less-than.
*/
static inline bool
-VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_IS_COMPRESSED(uint32 extinfo, int32 rawsize)
{
- return VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer) <
- (Size) (toast_pointer.va_rawsize - VARHDRSZ);
+ return VARATT_EXTINFO_GET_EXTSIZE(extinfo) < (Size) (rawsize - VARHDRSZ);
}
#endif
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index db45ee8881e..9e4f660836f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -4361,6 +4361,7 @@ timeout_params
timerCA
tlist_vinfo
toast_compress_header
+toast_external_data
tokenize_error_callback_arg
transferMode
transfer_thread_arg
@@ -4417,6 +4418,7 @@ vacuumingOptions
validate_string_relopt
varatt_expanded
varatt_external_oid
+varatt_external_oid8
varatt_indirect
varattrib_1b
varattrib_1b_e