public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v52 3/7] Make archiver process an auxiliary process
21+ messages / 8 participants
[nested] [flat]
* [PATCH v52 3/7] Make archiver process an auxiliary process
@ 2020-03-13 07:59 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 21+ messages in thread
From: Kyotaro Horiguchi @ 2020-03-13 07:59 UTC (permalink / raw)
This is a preliminary patch for shared-memory based stats collector.
Archiver process must be a auxiliary process since it uses shared
memory after stats data was moved into shared-memory. Make the process
an auxiliary process in order to make it work.
---
src/backend/access/transam/xlogarchive.c | 6 +-
src/backend/bootstrap/bootstrap.c | 22 ++-
src/backend/postmaster/pgarch.c | 224 ++++++++---------------
src/backend/postmaster/postmaster.c | 83 ++++-----
src/backend/storage/ipc/ipci.c | 2 +
src/include/miscadmin.h | 2 +
src/include/postmaster/pgarch.h | 13 +-
src/include/storage/pmsignal.h | 1 -
src/include/storage/proc.h | 8 +-
9 files changed, 139 insertions(+), 222 deletions(-)
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 1c5a4f8b5a..2558bcfb08 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -25,11 +25,13 @@
#include "common/archive.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "postmaster/pgarch.h"
#include "replication/walsender.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
-#include "storage/pmsignal.h"
+#include "storage/latch.h"
+#include "storage/proc.h"
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -491,7 +493,7 @@ XLogArchiveNotify(const char *xlog)
/* Notify archiver that it's got something to do */
if (IsUnderPostmaster)
- SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER);
+ PgArchWakeup();
}
/*
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index 6f615e6622..41da0c5059 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -317,6 +317,9 @@ AuxiliaryProcessMain(int argc, char *argv[])
case StartupProcess:
MyBackendType = B_STARTUP;
break;
+ case ArchiverProcess:
+ MyBackendType = B_ARCHIVER;
+ break;
case BgWriterProcess:
MyBackendType = B_BG_WRITER;
break;
@@ -437,30 +440,29 @@ AuxiliaryProcessMain(int argc, char *argv[])
proc_exit(1); /* should never return */
case StartupProcess:
- /* don't set signals, startup process has its own agenda */
StartupProcessMain();
- proc_exit(1); /* should never return */
+ proc_exit(1);
+
+ case ArchiverProcess:
+ PgArchiverMain();
+ proc_exit(1);
case BgWriterProcess:
- /* don't set signals, bgwriter has its own agenda */
BackgroundWriterMain();
- proc_exit(1); /* should never return */
+ proc_exit(1);
case CheckpointerProcess:
- /* don't set signals, checkpointer has its own agenda */
CheckpointerMain();
- proc_exit(1); /* should never return */
+ proc_exit(1);
case WalWriterProcess:
- /* don't set signals, walwriter has its own agenda */
InitXLOGAccess();
WalWriterMain();
- proc_exit(1); /* should never return */
+ proc_exit(1);
case WalReceiverProcess:
- /* don't set signals, walreceiver has its own agenda */
WalReceiverMain();
- proc_exit(1); /* should never return */
+ proc_exit(1);
default:
elog(PANIC, "unrecognized process type: %d", (int) MyAuxProcType);
diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c
index edec311f12..c399e6c267 100644
--- a/src/backend/postmaster/pgarch.c
+++ b/src/backend/postmaster/pgarch.c
@@ -38,16 +38,14 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
-#include "postmaster/fork_process.h"
#include "postmaster/interrupt.h"
#include "postmaster/pgarch.h"
-#include "postmaster/postmaster.h"
-#include "storage/dsm.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
-#include "storage/pg_shmem.h"
#include "storage/pmsignal.h"
+#include "storage/procsignal.h"
+#include "storage/spin.h"
#include "utils/guc.h"
#include "utils/ps_status.h"
@@ -73,153 +71,68 @@
*/
#define NUM_ORPHAN_CLEANUP_RETRIES 3
+/* Shared memory area for archiver process */
+typedef struct PgArchData
+{
+ Latch *latch; /* latch to wake the archiver up */
+ slock_t mutex; /* locks this struct */
+} PgArchData;
+
/* ----------
* Local data
* ----------
*/
-static time_t last_pgarch_start_time;
static time_t last_sigterm_time = 0;
+PgArchData *PgArch = NULL;
/*
* Flags set by interrupt handlers for later service in the main loop.
*/
-static volatile sig_atomic_t wakened = false;
static volatile sig_atomic_t ready_to_stop = false;
/* ----------
* Local function forward declarations
* ----------
*/
-#ifdef EXEC_BACKEND
-static pid_t pgarch_forkexec(void);
-#endif
-
-NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn();
-static void pgarch_waken(SIGNAL_ARGS);
static void pgarch_waken_stop(SIGNAL_ARGS);
static void pgarch_MainLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
static bool pgarch_readyXlog(char *xlog);
static void pgarch_archiveDone(char *xlog);
+static void pgarch_die(int code, Datum arg);
+/* Report shared memory space needed by PgArchShmemInit */
+Size
+PgArchShmemSize(void)
+{
+ Size size = 0;
-/* ------------------------------------------------------------
- * Public functions called from postmaster follow
- * ------------------------------------------------------------
- */
+ size = add_size(size, sizeof(PgArchData));
-/*
- * pgarch_start
- *
- * Called from postmaster at startup or after an existing archiver
- * died. Attempt to fire up a fresh archiver process.
- *
- * Returns PID of child process, or 0 if fail.
- *
- * Note: if fail, we will be called again from the postmaster main loop.
- */
-int
-pgarch_start(void)
-{
- time_t curtime;
- pid_t pgArchPid;
+ return size;
+}
- /*
- * Do nothing if no archiver needed
- */
- if (!XLogArchivingActive())
- return 0;
+/* Allocate and initialize archiver-related shared memory */
+void
+PgArchShmemInit(void)
+{
+ bool found;
- /*
- * Do nothing if too soon since last archiver start. This is a safety
- * valve to protect against continuous respawn attempts if the archiver is
- * dying immediately at launch. Note that since we will be re-called from
- * the postmaster main loop, we will get another chance later.
- */
- curtime = time(NULL);
- if ((unsigned int) (curtime - last_pgarch_start_time) <
- (unsigned int) PGARCH_RESTART_INTERVAL)
- return 0;
- last_pgarch_start_time = curtime;
+ PgArch = (PgArchData *)
+ ShmemInitStruct("Archiver ", PgArchShmemSize(), &found);
-#ifdef EXEC_BACKEND
- switch ((pgArchPid = pgarch_forkexec()))
-#else
- switch ((pgArchPid = fork_process()))
-#endif
+ if (!found)
{
- case -1:
- ereport(LOG,
- (errmsg("could not fork archiver: %m")));
- return 0;
-
-#ifndef EXEC_BACKEND
- case 0:
- /* in postmaster child ... */
- InitPostmasterChild();
-
- /* Close the postmaster's sockets */
- ClosePostmasterPorts(false);
-
- /* Drop our connection to postmaster's shared memory, as well */
- dsm_detach_all();
- PGSharedMemoryDetach();
-
- PgArchiverMain(0, NULL);
- break;
-#endif
-
- default:
- return (int) pgArchPid;
+ SpinLockInit(&PgArch->mutex);
+ PgArch->latch = NULL;
}
-
- /* shouldn't get here */
- return 0;
-}
-
-/* ------------------------------------------------------------
- * Local functions called by archiver follow
- * ------------------------------------------------------------
- */
-
-
-#ifdef EXEC_BACKEND
-
-/*
- * pgarch_forkexec() -
- *
- * Format up the arglist for, then fork and exec, archive process
- */
-static pid_t
-pgarch_forkexec(void)
-{
- char *av[10];
- int ac = 0;
-
- av[ac++] = "postgres";
-
- av[ac++] = "--forkarch";
-
- av[ac++] = NULL; /* filled in by postmaster_forkexec */
-
- av[ac] = NULL;
- Assert(ac < lengthof(av));
-
- return postmaster_forkexec(ac, av);
}
-#endif /* EXEC_BACKEND */
-
-/*
- * PgArchiverMain
- *
- * The argc/argv parameters are valid only in EXEC_BACKEND case. However,
- * since we don't use 'em, it hardly matters...
- */
-NON_EXEC_STATIC void
-PgArchiverMain(int argc, char *argv[])
+/* Main entry point for archiver process */
+void
+PgArchiverMain(void)
{
/*
* Ignore all signals usually bound to some action in the postmaster,
@@ -231,33 +144,48 @@ PgArchiverMain(int argc, char *argv[])
/* SIGQUIT handler was already set up by InitPostmasterChild */
pqsignal(SIGALRM, SIG_IGN);
pqsignal(SIGPIPE, SIG_IGN);
- pqsignal(SIGUSR1, pgarch_waken);
+ pqsignal(SIGUSR1, procsignal_sigusr1_handler);
pqsignal(SIGUSR2, pgarch_waken_stop);
+
/* Reset some signals that are accepted by postmaster but not here */
pqsignal(SIGCHLD, SIG_DFL);
+
+ /* Unblock signals (they were blocked when the postmaster forked us) */
PG_SETMASK(&UnBlockSig);
- MyBackendType = B_ARCHIVER;
- init_ps_display(NULL);
+ /* Arrange to clean up at archiver exit */
+ on_shmem_exit(pgarch_die, 0);
+
+ /*
+ * Advertise our latch that backends can use to wake us up while we're
+ * sleeping.
+ */
+ SpinLockAcquire(&PgArch->mutex);
+ PgArch->latch = MyLatch;
+ SpinLockRelease(&PgArch->mutex);
pgarch_MainLoop();
- exit(0);
+ proc_exit(0);
}
-/* SIGUSR1 signal handler for archiver process */
-static void
-pgarch_waken(SIGNAL_ARGS)
+/*
+ * Wake up the archiver
+ */
+void
+PgArchWakeup(void)
{
- int save_errno = errno;
+ Latch *latch;
- /* set flag that there is work to be done */
- wakened = true;
- SetLatch(MyLatch);
+ SpinLockAcquire(&PgArch->mutex);
+ latch = PgArch->latch;
+ SpinLockRelease(&PgArch->mutex);
- errno = save_errno;
+ if (latch)
+ SetLatch(latch);
}
+
/* SIGUSR2 signal handler for archiver process */
static void
pgarch_waken_stop(SIGNAL_ARGS)
@@ -282,14 +210,6 @@ pgarch_MainLoop(void)
pg_time_t last_copy_time = 0;
bool time_to_stop;
- /*
- * We run the copy loop immediately upon entry, in case there are
- * unarchived files left over from a previous database run (or maybe the
- * archiver died unexpectedly). After that we wait for a signal or
- * timeout before doing more.
- */
- wakened = true;
-
/*
* There shouldn't be anything for the archiver to do except to wait for a
* signal ... however, the archiver exists to protect our data, so she
@@ -328,12 +248,8 @@ pgarch_MainLoop(void)
}
/* Do what we're here for */
- if (wakened || time_to_stop)
- {
- wakened = false;
- pgarch_ArchiverCopyLoop();
- last_copy_time = time(NULL);
- }
+ pgarch_ArchiverCopyLoop();
+ last_copy_time = time(NULL);
/*
* Sleep until a signal is received, or until a poll is forced by
@@ -354,13 +270,9 @@ pgarch_MainLoop(void)
WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
timeout * 1000L,
WAIT_EVENT_ARCHIVER_MAIN);
- if (rc & WL_TIMEOUT)
- wakened = true;
if (rc & WL_POSTMASTER_DEATH)
time_to_stop = true;
}
- else
- wakened = true;
}
/*
@@ -744,3 +656,17 @@ pgarch_archiveDone(char *xlog)
StatusFilePath(rlogdone, xlog, ".done");
(void) durable_rename(rlogready, rlogdone, WARNING);
}
+
+
+/*
+ * pgarch_die
+ *
+ * exit-time cleanup handler
+ */
+static void
+pgarch_die(int code, Datum arg)
+{
+ SpinLockAcquire(&PgArch->mutex);
+ PgArch->latch = NULL;
+ SpinLockRelease(&PgArch->mutex);
+}
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index edab95a19e..34c8551288 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -548,6 +548,7 @@ static void ShmemBackendArrayRemove(Backend *bn);
#endif /* EXEC_BACKEND */
#define StartupDataBase() StartChildProcess(StartupProcess)
+#define StartArchiver() StartChildProcess(ArchiverProcess)
#define StartBackgroundWriter() StartChildProcess(BgWriterProcess)
#define StartCheckpointer() StartChildProcess(CheckpointerProcess)
#define StartWalWriter() StartChildProcess(WalWriterProcess)
@@ -1792,7 +1793,7 @@ ServerLoop(void)
/* If we have lost the archiver, try to start a new one. */
if (PgArchPID == 0 && PgArchStartupAllowed())
- PgArchPID = pgarch_start();
+ PgArchPID = StartArchiver();
/* If we need to signal the autovacuum launcher, do so now */
if (avlauncher_needs_signal)
@@ -3007,7 +3008,7 @@ reaper(SIGNAL_ARGS)
if (!IsBinaryUpgrade && AutoVacuumingActive() && AutoVacPID == 0)
AutoVacPID = StartAutoVacLauncher();
if (PgArchStartupAllowed() && PgArchPID == 0)
- PgArchPID = pgarch_start();
+ PgArchPID = StartArchiver();
if (PgStatPID == 0)
PgStatPID = pgstat_start();
@@ -3141,24 +3142,6 @@ reaper(SIGNAL_ARGS)
continue;
}
- /*
- * Was it the archiver? If so, just try to start a new one; no need
- * to force reset of the rest of the system. (If fail, we'll try
- * again in future cycles of the main loop.). Unless we were waiting
- * for it to shut down; don't restart it in that case, and
- * PostmasterStateMachine() will advance to the next shutdown step.
- */
- if (pid == PgArchPID)
- {
- PgArchPID = 0;
- if (!EXIT_STATUS_0(exitstatus))
- LogChildExit(LOG, _("archiver process"),
- pid, exitstatus);
- if (PgArchStartupAllowed())
- PgArchPID = pgarch_start();
- continue;
- }
-
/*
* Was it the statistics collector? If so, just try to start a new
* one; no need to force reset of the rest of the system. (If fail,
@@ -3175,6 +3158,26 @@ reaper(SIGNAL_ARGS)
continue;
}
+ /*
+ * Was it the archiver? If exit status is zero (normal) or one (FATAL
+ * exit), we assume everything is all right just like normal backends
+ * and just try to restart a new one so that we immediately retry
+ * archiving remaining files. (If fail, we'll try again in future
+ * cycles of the postmaster's main loop.) Unless we were waiting for it
+ * to shut down; don't restart it in that case, and
+ * PostmasterStateMachine() will advance to the next shutdown step.
+ */
+ if (pid == PgArchPID)
+ {
+ PgArchPID = 0;
+ if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus))
+ HandleChildCrash(pid, exitstatus,
+ _("archiver process"));
+ if (PgArchStartupAllowed())
+ PgArchPID = StartArchiver();
+ continue;
+ }
+
/* Was it the system logger? If so, try to start a new one */
if (pid == SysLoggerPID)
{
@@ -3403,7 +3406,7 @@ CleanupBackend(int pid,
/*
* HandleChildCrash -- cleanup after failed backend, bgwriter, checkpointer,
- * walwriter, autovacuum, or background worker.
+ * walwriter, autovacuum, archiver or background worker.
*
* The objectives here are to clean up our local state about the child
* process, and to signal all other remaining children to quickdie.
@@ -3609,19 +3612,16 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
signal_child(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT));
}
- /*
- * Force a power-cycle of the pgarch process too. (This isn't absolutely
- * necessary, but it seems like a good idea for robustness, and it
- * simplifies the state-machine logic in the case where a shutdown request
- * arrives during crash processing.)
- */
- if (PgArchPID != 0 && take_action)
+ /* Take care of the archiver too */
+ if (pid == PgArchPID)
+ PgArchPID = 0;
+ else if (PgArchPID != 0 && take_action)
{
ereport(DEBUG2,
(errmsg_internal("sending %s to process %d",
- "SIGQUIT",
+ (SendStop ? "SIGSTOP" : "SIGQUIT"),
(int) PgArchPID)));
- signal_child(PgArchPID, SIGQUIT);
+ signal_child(PgArchPID, (SendStop ? SIGSTOP : SIGQUIT));
}
/*
@@ -3912,6 +3912,7 @@ PostmasterStateMachine(void)
Assert(CheckpointerPID == 0);
Assert(WalWriterPID == 0);
Assert(AutoVacPID == 0);
+ Assert(PgArchPID == 0);
/* syslogger is not considered here */
pmState = PM_NO_CHILDREN;
}
@@ -5037,12 +5038,6 @@ SubPostmasterMain(int argc, char *argv[])
StartBackgroundWorker();
}
- if (strcmp(argv[1], "--forkarch") == 0)
- {
- /* Do not want to attach to shared memory */
-
- PgArchiverMain(argc, argv); /* does not return */
- }
if (strcmp(argv[1], "--forkcol") == 0)
{
/* Do not want to attach to shared memory */
@@ -5140,7 +5135,7 @@ sigusr1_handler(SIGNAL_ARGS)
*/
Assert(PgArchPID == 0);
if (XLogArchivingAlways())
- PgArchPID = pgarch_start();
+ PgArchPID = StartArchiver();
/*
* If we aren't planning to enter hot standby mode later, treat
@@ -5194,16 +5189,6 @@ sigusr1_handler(SIGNAL_ARGS)
if (StartWorkerNeeded || HaveCrashedWorker)
maybe_start_bgworkers();
- if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER) &&
- PgArchPID != 0)
- {
- /*
- * Send SIGUSR1 to archiver process, to wake it up and begin archiving
- * next WAL file.
- */
- signal_child(PgArchPID, SIGUSR1);
- }
-
/* Tell syslogger to rotate logfile if requested */
if (SysLoggerPID != 0)
{
@@ -5445,6 +5430,10 @@ StartChildProcess(AuxProcType type)
ereport(LOG,
(errmsg("could not fork startup process: %m")));
break;
+ case ArchiverProcess:
+ ereport(LOG,
+ (errmsg("could not fork archiver process: %m")));
+ break;
case BgWriterProcess:
ereport(LOG,
(errmsg("could not fork background writer process: %m")));
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index f9bbe97b50..3e4ec53a97 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -144,6 +144,7 @@ CreateSharedMemoryAndSemaphores(void)
size = add_size(size, ReplicationOriginShmemSize());
size = add_size(size, WalSndShmemSize());
size = add_size(size, WalRcvShmemSize());
+ size = add_size(size, PgArchShmemSize());
size = add_size(size, ApplyLauncherShmemSize());
size = add_size(size, SnapMgrShmemSize());
size = add_size(size, BTreeShmemSize());
@@ -258,6 +259,7 @@ CreateSharedMemoryAndSemaphores(void)
ReplicationOriginShmemInit();
WalSndShmemInit();
WalRcvShmemInit();
+ PgArchShmemInit();
ApplyLauncherShmemInit();
/*
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 1bdc97e308..adb9f819bb 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -419,6 +419,7 @@ typedef enum
BootstrapProcess,
StartupProcess,
BgWriterProcess,
+ ArchiverProcess,
CheckpointerProcess,
WalWriterProcess,
WalReceiverProcess,
@@ -431,6 +432,7 @@ extern AuxProcType MyAuxProcType;
#define AmBootstrapProcess() (MyAuxProcType == BootstrapProcess)
#define AmStartupProcess() (MyAuxProcType == StartupProcess)
#define AmBackgroundWriterProcess() (MyAuxProcType == BgWriterProcess)
+#define AmArchiverProcess() (MyAuxProcType == ArchiverProcess)
#define AmCheckpointerProcess() (MyAuxProcType == CheckpointerProcess)
#define AmWalWriterProcess() (MyAuxProcType == WalWriterProcess)
#define AmWalReceiverProcess() (MyAuxProcType == WalReceiverProcess)
diff --git a/src/include/postmaster/pgarch.h b/src/include/postmaster/pgarch.h
index d102a21ab7..d053bf1564 100644
--- a/src/include/postmaster/pgarch.h
+++ b/src/include/postmaster/pgarch.h
@@ -26,14 +26,9 @@
#define MAX_XFN_CHARS 40
#define VALID_XFN_CHARS "0123456789ABCDEF.history.backup.partial"
-/* ----------
- * Functions called from postmaster
- * ----------
- */
-extern int pgarch_start(void);
-
-#ifdef EXEC_BACKEND
-extern void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn();
-#endif
+extern Size PgArchShmemSize(void);
+extern void PgArchShmemInit(void);
+extern void PgArchiverMain(void) pg_attribute_noreturn();
+extern void PgArchWakeup(void);
#endif /* _PGARCH_H */
diff --git a/src/include/storage/pmsignal.h b/src/include/storage/pmsignal.h
index dbbed18f61..8ed4d87ae6 100644
--- a/src/include/storage/pmsignal.h
+++ b/src/include/storage/pmsignal.h
@@ -34,7 +34,6 @@ typedef enum
{
PMSIGNAL_RECOVERY_STARTED, /* recovery has started */
PMSIGNAL_BEGIN_HOT_STANDBY, /* begin Hot Standby */
- PMSIGNAL_WAKEN_ARCHIVER, /* send a NOTIFY signal to xlog archiver */
PMSIGNAL_ROTATE_LOGFILE, /* send SIGUSR1 to syslogger to rotate logfile */
PMSIGNAL_START_AUTOVAC_LAUNCHER, /* start an autovacuum launcher */
PMSIGNAL_START_AUTOVAC_WORKER, /* start an autovacuum worker */
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index a777cb64a1..2fd1ff09a7 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -370,11 +370,11 @@ extern PGPROC *PreparedXactProcs;
* We set aside some extra PGPROC structures for auxiliary processes,
* ie things that aren't full-fledged backends but need shmem access.
*
- * Background writer, checkpointer and WAL writer run during normal operation.
- * Startup process and WAL receiver also consume 2 slots, but WAL writer is
- * launched only after startup has exited, so we only need 4 slots.
+ * Background writer, checkpointer, WAL writer and archiver run during normal
+ * operation. Startup process and WAL receiver also consume 2 slots, but WAL
+ * writer is launched only after startup has exited, so we only need 5 slots.
*/
-#define NUM_AUXILIARY_PROCS 4
+#define NUM_AUXILIARY_PROCS 5
/* configurable options */
extern PGDLLIMPORT int DeadlockTimeout;
--
2.27.0
----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v52-0004-Shared-memory-based-stats-collector.patch"
^ permalink raw reply [nested|flat] 21+ messages in thread
* ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 05:32 Nikhil Kumar Veldanda <[email protected]>
0 siblings, 3 replies; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-03-06 05:32 UTC (permalink / raw)
To: pgsql-hackers
Hi all,
The ZStandard compression algorithm [1][2], though not currently used for
TOAST compression in PostgreSQL, offers significantly improved compression
ratios compared to lz4/pglz in both dictionary-based and non-dictionary
modes. Attached find for review my patch to add ZStandard compression to
Postgres. In tests this patch used with a pre-trained dictionary achieved
up to four times the compression ratio of LZ4, while ZStandard without a
dictionary outperformed LZ4/pglz by about two times during compression of
data.
Notably, this is the first compression algorithm for Postgres that can make
use of a dictionary to provide higher levels of compression, but
dictionaries have to be generated and maintained, and so I’ve had to break
new ground in that regard. To use the dictionary support requires training
and storing a dictionary for a given variable-length column type. On a
variable-length column, a SQL function will be called. It will sample the
column’s data and feed it into the ZStandard training API which will return
a dictionary. In the example, the column is of JSONB type. The SQL function
takes the table name and the attribute number as inputs. If the training is
successful, it will return true; otherwise, it will return false.
‘’‘
test=# select build_zstd_dict_for_attribute('"public"."zstd"', 1);
build_zstd_dict_for_attribute
-------------------------------
t
(1 row)
‘’‘
The sampling logic and data to feed to the ZStandard training API can vary
by data type. The patch includes an method to write other type-specific
training functions and includes a default for JSONB, TEXT and BYTEA. There
is a new option called ‘build_zstd_dict’ that takes a function name as
input in ‘CREATE TYPE’. In this way anyone can write their own
type-specific training function by handling sampling logic and returning
the necessary information for the ZStandard training API in
“ZstdTrainingData” format.
```
typedef struct ZstdTrainingData
{
char *sample_buffer; /* Pointer to the raw sample buffer */
size_t *sample_sizes; /* Array of sample sizes */
int nitems; /* Number of sample sizes */
} ZstdTrainingData;
```
This information is feed into the ZStandard train API, which generates a
dictionary and inserts it into the dictionary catalog table. Additionally,
we update the ‘pg_attribute’ attribute options to include the unique
dictionary ID for that specific attribute. During compression, based on the
available dictionary ID, we retrieve the dictionary and use it to compress
the documents. I’ve created standard training function
(`zstd_dictionary_builder`) for JSONB, TEXT, and BYTEA.
We store dictionary and dictid in the new catalog table
‘pg_zstd_dictionaries’
```
test=# \d pg_zstd_dictionaries
Table "pg_catalog.pg_zstd_dictionaries"
Column | Type | Collation | Nullable | Default
--------+-------+-----------+----------+---------
dictid | oid | | not null |
dict | bytea | | not null |
Indexes:
"pg_zstd_dictionaries_dictid_index" PRIMARY KEY, btree (dictid)
```
This is the entire ZStandard dictionary infrastructure. A column can have
multiple dictionaries. The latest dictionary will be identified by the
pg_attribute attoptions. We never delete dictionaries once they are
generated. If a dictionary is not provided and attcompression is set to
zstd, we compress with ZStandard without dictionary. For decompression, the
zstd-compressed frame contains a dictionary identifier (dictid) that
indicates the dictionary used for compression. By retrieving this dictid
from the zstd frame, we then fetch the corresponding dictionary and perform
decompression.
#############################################################################
Enter toast compression framework changes,
We identify a compressed datum compression algorithm using the top two bits
of va_tcinfo (varattrib_4b.va_compressed).
It is possible to have four compression methods. However, based on previous
community email discussions regarding toast compression changes[3], the
idea of using it for a new compression algorithm has been rejected, and a
suggestion has been made to extend it which I’ve implemented in this patch.
This change necessitates an update to ‘varattrib_4b’ and ‘varatt_external’
on disk structures. I’ve made sure that this changes are backward
compatible.
```
typedef union
{
struct /* Normal varlena (4-byte length) */
{
uint32 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_4byte;
struct /* Compressed-in-line format */
{
uint32 va_header;
uint32 va_tcinfo; /* Original data size (excludes header) and
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
struct
{
uint32 va_header;
uint32 va_tcinfo;
uint32 va_cmp_alg;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_compressed_ext;
} varattrib_4b;
typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
uint32 va_extinfo; /* External saved size (without header) and
* compression method */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
uint32 va_cmp_alg; /* The additional compression algorithms
* information. */
} varatt_external;
```
As I need to update this structs, I’ve made changes to the existing macros.
Additionally added compression and decompression routines related to
ZStandard as needed. These are major design changes in the patch to
incorporate ZStandard with dictionary compression.
Please let me know what you think about all this. Are there any concerns
with my approach? In particular, I would appreciate your thoughts on the
on-disk changes that result from this.
kind regards,
Nikhil Veldanda
Amazon Web Services: https://aws.amazon.com
[1] https://facebook.github.io/zstd/
[2] https://github.com/facebook/zstd
[3]
https://www.postgresql.org/message-id/flat/YoMiNmkztrslDbNS%40paquier.xyz
Attachments:
[application/octet-stream] v1-0001-Add-ZStandard-with-dictionaries-compression-suppo.patch (106.5K, ../../CAFAfj_HX84EK4hyRYw50AOHOcdVi-+FFwAAPo7JHx4aShCvunQ@mail.gmail.com/3-v1-0001-Add-ZStandard-with-dictionaries-compression-suppo.patch)
download | inline diff:
From 94cbf115d2d12d3908eac5b784c608e16df579e6 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Tue, 4 Mar 2025 08:14:32 +0000
Subject: [PATCH v1] Add ZStandard (with dictionaries) compression support for
TOAST
---
contrib/amcheck/verify_heapam.c | 1 +
doc/src/sgml/catalogs.sgml | 55 ++
doc/src/sgml/ref/create_type.sgml | 21 +-
src/backend/access/brin/brin_tuple.c | 18 +-
src/backend/access/common/detoast.c | 12 +-
src/backend/access/common/indextuple.c | 18 +-
src/backend/access/common/reloptions.c | 36 +-
src/backend/access/common/toast_compression.c | 269 +++++++-
src/backend/access/common/toast_internals.c | 8 +-
src/backend/access/table/toast_helper.c | 17 +-
src/backend/catalog/Makefile | 3 +-
src/backend/catalog/heap.c | 8 +-
src/backend/catalog/meson.build | 1 +
src/backend/catalog/pg_type.c | 11 +-
src/backend/catalog/pg_zstd_dictionaries.c | 601 ++++++++++++++++++
src/backend/commands/analyze.c | 7 +-
src/backend/commands/typecmds.c | 99 ++-
src/backend/utils/adt/varlena.c | 3 +
src/backend/utils/misc/guc_tables.c | 3 +
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/bin/pg_amcheck/t/004_verify_heapam.pl | 14 +-
src/bin/pg_dump/pg_dump.c | 25 +-
src/bin/psql/describe.c | 5 +-
src/include/access/toast_compression.h | 13 +-
src/include/access/toast_helper.h | 2 +
src/include/access/toast_internals.h | 35 +-
src/include/catalog/Makefile | 3 +-
src/include/catalog/catversion.h | 2 +-
src/include/catalog/meson.build | 1 +
src/include/catalog/pg_proc.dat | 10 +
src/include/catalog/pg_type.dat | 6 +-
src/include/catalog/pg_type.h | 8 +-
src/include/catalog/pg_zstd_dictionaries.h | 53 ++
src/include/parser/analyze.h | 5 +
src/include/utils/attoptcache.h | 6 +
src/include/varatt.h | 80 ++-
src/test/regress/expected/compression.out | 5 +-
src/test/regress/expected/compression_1.out | 3 +
.../regress/expected/compression_zstd.out | 123 ++++
.../regress/expected/compression_zstd_1.out | 181 ++++++
src/test/regress/expected/oidjoins.out | 1 +
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/compression.sql | 1 +
src/test/regress/sql/compression_zstd.sql | 97 +++
src/tools/pgindent/typedefs.list | 5 +
45 files changed, 1791 insertions(+), 88 deletions(-)
create mode 100644 src/backend/catalog/pg_zstd_dictionaries.c
create mode 100644 src/include/catalog/pg_zstd_dictionaries.h
create mode 100644 src/test/regress/expected/compression_zstd.out
create mode 100644 src/test/regress/expected/compression_zstd_1.out
create mode 100644 src/test/regress/sql/compression_zstd.sql
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 827312306f..f01cc940e3 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1700,6 +1700,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* List of all valid compression method IDs */
case TOAST_PGLZ_COMPRESSION_ID:
case TOAST_LZ4_COMPRESSION_ID:
+ case TOAST_ZSTD_COMPRESSION_ID:
valid = true;
break;
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index fb05063555..ed4c51a678 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -369,6 +369,12 @@
<entry><link linkend="catalog-pg-user-mapping"><structname>pg_user_mapping</structname></link></entry>
<entry>mappings of users to foreign servers</entry>
</row>
+
+ <row>
+ <entry><link linkend="catalog-pg-zstd-dictionaries"><structname>pg_zstd_dictionaries</structname></link></entry>
+ <entry>Zstandard dictionaries</entry>
+ </row>
+
</tbody>
</tgroup>
</table>
@@ -9779,4 +9785,53 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</table>
</sect1>
+
+<sect1 id="catalog-pg-zstd-dictionaries">
+ <title><structname>pg_zstd_dictionaries</structname></title>
+
+ <indexterm zone="catalog-pg-zstd-dictionaries">
+ <primary>pg_zstd_dictionaries</primary>
+ </indexterm>
+
+ <para>
+ The catalog <structname>pg_zstd_dictionaries</structname> maintains the dictionaries essential for Zstandard compression and decompression.
+ </para>
+
+ <table>
+ <title><structname>pg_zstd_dictionaries</structname> Columns</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">Column Type</para>
+ <para>Description</para>
+ </entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dictid</structfield> <type>oid</type>
+ </para>
+ <para>
+ Dictionary identifier; a non-null OID that uniquely identifies a dictionary.
+ </para>
+ </entry>
+ </row>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dict</structfield> <type>bytea</type>
+ </para>
+ <para>
+ Variable-length field containing the zstd dictionary data. This field must not be null.
+ </para>
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+</sect1>
+
</chapter>
diff --git a/doc/src/sgml/ref/create_type.sgml b/doc/src/sgml/ref/create_type.sgml
index 994dfc6526..ad4cf2f8b3 100644
--- a/doc/src/sgml/ref/create_type.sgml
+++ b/doc/src/sgml/ref/create_type.sgml
@@ -56,6 +56,7 @@ CREATE TYPE <replaceable class="parameter">name</replaceable> (
[ , ELEMENT = <replaceable class="parameter">element</replaceable> ]
[ , DELIMITER = <replaceable class="parameter">delimiter</replaceable> ]
[ , COLLATABLE = <replaceable class="parameter">collatable</replaceable> ]
+ [ , BUILD_ZSTD_DICT = <replaceable class="parameter">zstd_training_function</replaceable> ]
)
CREATE TYPE <replaceable class="parameter">name</replaceable>
@@ -211,7 +212,8 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
<replaceable class="parameter">type_modifier_input_function</replaceable>,
<replaceable class="parameter">type_modifier_output_function</replaceable>,
<replaceable class="parameter">analyze_function</replaceable>, and
- <replaceable class="parameter">subscript_function</replaceable>
+ <replaceable class="parameter">subscript_function</replaceable>, and
+ <replaceable class="parameter">zstd_training_function</replaceable>
are optional. Generally these functions have to be coded in C
or another low-level language.
</para>
@@ -491,6 +493,15 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
make use of the collation information; this does not happen
automatically merely by marking the type collatable.
</para>
+
+ <para>
+ The optional <replaceable class="parameter">zstd_training_function</replaceable>
+ performs type-specific sample collection for a column of the corresponding data type.
+ By default, for <type>jsonb</type>, <type>text</type>, and <type>bytea</type> data types, the function <literal>zstd_dictionary_builder</literal> is defined. It attempts to gather samples for a column
+ and returns a sample buffer for zstd dictionary training. The training function must be declared to accept two arguments of type <type>internal</type> and return an <type>internal</type> result.
+ The detailed information for zstd training function is provided in <filename>src/backend/catalog/pg_zstd_dictionaries.c</filename>.
+ </para>
+
</refsect2>
<refsect2 id="sql-createtype-array" xreflabel="Array Types">
@@ -846,6 +857,14 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
</para>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><replaceable class="parameter">build_zstd_dict</replaceable></term>
+ <listitem>
+ <para>
+ Specifies the name of a function that performs sampling and provides the logic necessary to generate a sample buffer for zstd training.
+ </para>
+ </listitem>
+ </varlistentry>
</variablelist>
</refsect1>
diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c
index 861f397e6d..67175942e9 100644
--- a/src/backend/access/brin/brin_tuple.c
+++ b/src/backend/access/brin/brin_tuple.c
@@ -40,7 +40,7 @@
#include "access/tupmacs.h"
#include "utils/datum.h"
#include "utils/memutils.h"
-
+#include "utils/attoptcache.h"
/*
* This enables de-toasting of index entries. Needed until VACUUM is
@@ -223,6 +223,8 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
{
Datum cvalue;
char compression;
+ int zstd_cmp_level = DEFAULT_ZSTD_CMP_LEVEL;
+ Oid zstd_dictid = InvalidDictId;
Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
keyno);
@@ -237,7 +239,19 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
else
compression = InvalidCompressionMethod;
- cvalue = toast_compress_datum(value, compression);
+ if (compression == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ {
+ zstd_cmp_level = aopt->zstd_cmp_level;
+ zstd_dictid = (Oid) aopt->zstd_dictid;
+ }
+ }
+ cvalue = toast_compress_datum(value, compression,
+ zstd_dictid,
+ zstd_cmp_level);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 6265178774..b57a9f024c 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr,
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or Zstandard data, we'll have to
+ * fetch the whole thing, because there doesn't seem to be an API
+ * call to determine how much compressed data we need to be sure
+ * of being able to decompress the required slice.
*/
if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
TOAST_PGLZ_COMPRESSION_ID)
@@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index 1986b943a2..625f91629c 100644
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -21,6 +21,7 @@
#include "access/htup_details.h"
#include "access/itup.h"
#include "access/toast_internals.h"
+#include "utils/attoptcache.h"
/*
* This enables de-toasting of index entries. Needed until VACUUM is
@@ -124,8 +125,23 @@ index_form_tuple_context(TupleDesc tupleDescriptor,
{
Datum cvalue;
+ int zstd_cmp_level = DEFAULT_ZSTD_CMP_LEVEL;
+ Oid zstd_dictid = InvalidDictId;
+
+ if (att->attcompression == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ {
+ zstd_cmp_level = aopt->zstd_cmp_level;
+ zstd_dictid = (Oid) aopt->zstd_dictid;
+ }
+ }
cvalue = toast_compress_datum(untoasted_values[i],
- att->attcompression);
+ att->attcompression,
+ zstd_dictid,
+ zstd_cmp_level);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 59fb53e770..b99ba93ec3 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -34,6 +34,7 @@
#include "utils/guc.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "access/toast_compression.h"
/*
* Contents of pg_class.reloptions
@@ -389,7 +390,26 @@ static relopt_int intRelOpts[] =
},
-1, 0, 1024
},
-
+ {
+ {
+ "zstd_dict_size",
+ "Max dict size for zstd",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_DICT_SIZE, 0, 112640 /* Max dict size(110 KB), 0
+ * indicates Don't use dictionary
+ * for compression */
+ },
+ {
+ {
+ "zstd_cmp_level",
+ "Set column's ZSTD compression level",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_CMP_LEVEL, 1, 22
+ },
/* list terminator */
{{NULL}}
};
@@ -478,6 +498,15 @@ static relopt_real realRelOpts[] =
},
0, -1.0, DBL_MAX
},
+ {
+ {
+ "zstd_dictid",
+ "Current Zstd dictid for column",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ InvalidDictId, InvalidDictId, UINT32_MAX
+ },
{
{
"vacuum_cleanup_index_scale_factor",
@@ -2093,7 +2122,10 @@ attribute_reloptions(Datum reloptions, bool validate)
{
static const relopt_parse_elt tab[] = {
{"n_distinct", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct)},
- {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)}
+ {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)},
+ {"zstd_dictid", RELOPT_TYPE_REAL, offsetof(AttributeOpts, zstd_dictid)},
+ {"zstd_dict_size", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_dict_size)},
+ {"zstd_cmp_level", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_cmp_level)},
};
return (bytea *) build_reloptions(reloptions, validate,
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 21f2f4af97..64151eb778 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -17,19 +17,25 @@
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#include <zdict.h>
+#endif
+
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"
+#include "catalog/pg_zstd_dictionaries.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
-#define NO_LZ4_SUPPORT() \
+#define NO_METHOD_SUPPORT(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with %s support.", method)))
/*
* Compress a varlena using PGLZ.
@@ -139,7 +145,7 @@ struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 valsize;
@@ -182,7 +188,7 @@ struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -215,7 +221,7 @@ struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -289,10 +295,17 @@ CompressionNameToMethod(const char *compression)
else if (strcmp(compression, "lz4") == 0)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd") == 0)
+ {
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+#endif
+ return TOAST_ZSTD_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -309,8 +322,250 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_COMPRESSION:
+ return "zstd";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
}
}
+
+/* Compress datum using ZSTD with optional dictionary (using cdict) */
+struct varlena *
+zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level)
+{
+#ifdef USE_ZSTD
+ uint32 valsize = VARSIZE_ANY_EXHDR(value);
+ size_t max_size = ZSTD_compressBound(valsize);
+ struct varlena *compressed;
+ void *dest;
+ size_t cmp_size,
+ ret;
+ ZSTD_CCtx *cctx = ZSTD_createCCtx();
+ ZSTD_CDict *cdict = NULL;
+
+ if (!cctx)
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression context")));
+
+ ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD compression level: %s", ZSTD_getErrorName(ret))));
+ }
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ cdict = ZSTD_createCDict(dict_buffer, dict_size, zstd_level);
+ pfree(dict_bytea);
+
+ if (!cdict)
+ {
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_CCtx_refCDict(cctx, cdict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeCDict(cdict);
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ /* Allocate space for the compressed varlena (header + data) */
+ compressed = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED_EXT);
+ dest = (char *) compressed + VARHDRSZ_COMPRESSED_EXT;
+
+ /* Compress the data */
+ cmp_size = ZSTD_compress2(cctx, dest, max_size, VARDATA_ANY(value), valsize);
+
+ /* Cleanup */
+ ZSTD_freeCDict(cdict);
+ ZSTD_freeCCtx(cctx);
+
+ if (ZSTD_isError(cmp_size))
+ {
+ pfree(compressed);
+ ereport(ERROR, (errmsg("ZSTD compression failed: %s", ZSTD_getErrorName(cmp_size))));
+ }
+
+ /*
+ * If compression did not reduce size, return NULL so that the
+ * uncompressed data is stored
+ */
+ if (cmp_size > valsize)
+ {
+ pfree(compressed);
+ return NULL;
+ }
+
+ /* Set the compressed size in the varlena header */
+ SET_VARSIZE_COMPRESSED(compressed, cmp_size + VARHDRSZ_COMPRESSED_EXT);
+ return compressed;
+
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
+
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifdef USE_ZSTD
+ uint32 actual_size_exhdr = VARDATA_COMPRESSED_GET_EXTSIZE(value);
+ uint32 cmp_size_exhdr = VARSIZE_4B(value) - VARHDRSZ_COMPRESSED_EXT;
+ Oid dictid;
+ struct varlena *result;
+ size_t uncmp_size,
+ ret;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+ ZSTD_DDict *ddict = NULL;
+
+ if (!dctx)
+ ereport(ERROR, (errmsg("Failed to create ZSTD decompression context")));
+
+ /*
+ * Extract the dictionary ID from the compressed frame. This function
+ * reads the dictionary ID from the frame header.
+ */
+ dictid = (Oid) ZSTD_getDictID_fromFrame(VARDATA_4B_C(value), cmp_size_exhdr);
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ ddict = ZSTD_createDDict(dict_buffer, dict_size);
+ pfree(dict_bytea);
+
+ if (!ddict)
+ {
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_DCtx_refDDict(dctx, ddict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ /* Allocate space for the uncompressed data */
+ result = (struct varlena *) palloc(actual_size_exhdr + VARHDRSZ);
+
+ uncmp_size = ZSTD_decompressDCtx(dctx,
+ VARDATA(result),
+ actual_size_exhdr,
+ VARDATA_4B_C(value),
+ cmp_size_exhdr);
+
+ /* Cleanup */
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+
+ if (ZSTD_isError(uncmp_size))
+ {
+ pfree(result);
+ ereport(ERROR, (errmsg("ZSTD decompression failed: %s", ZSTD_getErrorName(uncmp_size))));
+ }
+
+ /* Set final size in the varlena header */
+ SET_VARSIZE(result, uncmp_size + VARHDRSZ);
+ return result;
+
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
+
+/* Decompress a slice of the datum using the streaming API and optional dictionary */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+#ifdef USE_ZSTD
+ struct varlena *result;
+ ZSTD_inBuffer inBuf;
+ ZSTD_outBuffer outBuf;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+ ZSTD_DDict *ddict = NULL;
+ Oid dictid;
+ uint32 cmp_size_exhdr = VARSIZE_4B(value) - VARHDRSZ_COMPRESSED_EXT;
+ size_t ret;
+
+ if (dctx == NULL)
+ elog(ERROR, "could not create zstd decompression context");
+
+ /* Extract the dictionary ID from the compressed frame */
+ dictid = (Oid) ZSTD_getDictID_fromFrame(VARDATA_4B_C(value), cmp_size_exhdr);
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ /* Create and bind the dictionary to the decompression context */
+ ddict = ZSTD_createDDict(dict_buffer, dict_size);
+ pfree(dict_bytea);
+
+ if (!ddict)
+ {
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_DCtx_refDDict(dctx, ddict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ inBuf.src = (char *) value + VARHDRSZ_COMPRESSED_EXT;
+ inBuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED_EXT;
+ inBuf.pos = 0;
+
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+ outBuf.dst = (char *) result + VARHDRSZ;
+ outBuf.size = slicelength;
+ outBuf.pos = 0;
+
+ /* Common decompression loop */
+ while (inBuf.pos < inBuf.size && outBuf.pos < outBuf.size)
+ {
+ ret = ZSTD_decompressStream(dctx, &outBuf, &inBuf);
+ if (ZSTD_isError(ret))
+ {
+ pfree(result);
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ elog(ERROR, "zstd decompression failed: %s", ZSTD_getErrorName(ret));
+ }
+ }
+
+ /* Cleanup */
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+
+ Assert(outBuf.size == slicelength && outBuf.pos == slicelength);
+ SET_VARSIZE(result, outBuf.pos + VARHDRSZ);
+ return result;
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 7d8be8346c..c8a03a6aab 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -43,7 +43,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
* ----------
*/
Datum
-toast_compress_datum(Datum value, char cmethod)
+toast_compress_datum(Datum value, char cmethod, Oid zstd_dictid, int zstd_level)
{
struct varlena *tmp = NULL;
int32 valsize;
@@ -71,6 +71,10 @@ toast_compress_datum(Datum value, char cmethod)
tmp = lz4_compress_datum((const struct varlena *) value);
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_COMPRESSION:
+ tmp = zstd_compress_datum((const struct varlena *) value, zstd_dictid, zstd_level);
+ cmid = TOAST_ZSTD_COMPRESSION_ID;
+ break;
default:
elog(ERROR, "invalid compression method %c", cmethod);
}
@@ -176,6 +180,7 @@ toast_save_datum(Relation rel, Datum value,
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
toast_pointer.va_extinfo = data_todo;
+ toast_pointer.va_cmp_alg = 0;
}
else if (VARATT_IS_COMPRESSED(dval))
{
@@ -196,6 +201,7 @@ toast_save_datum(Relation rel, Datum value,
data_todo = VARSIZE(dval) - VARHDRSZ;
toast_pointer.va_rawsize = VARSIZE(dval);
toast_pointer.va_extinfo = data_todo;
+ toast_pointer.va_cmp_alg = 0;
}
/*
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index b60fab0a4d..bba90097d3 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -19,7 +19,8 @@
#include "access/toast_internals.h"
#include "catalog/pg_type_d.h"
#include "varatt.h"
-
+#include "utils/attoptcache.h"
+#include "access/toast_compression.h"
/*
* Prepare to TOAST a tuple.
@@ -55,6 +56,18 @@ toast_tuple_init(ToastTupleContext *ttc)
ttc->ttc_attr[i].tai_colflags = 0;
ttc->ttc_attr[i].tai_oldexternal = NULL;
ttc->ttc_attr[i].tai_compression = att->attcompression;
+ ttc->ttc_attr[i].zstd_dictid = InvalidDictId;
+ ttc->ttc_attr[i].zstd_level = DEFAULT_ZSTD_CMP_LEVEL;
+ if (att->attcompression == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt)
+ {
+ ttc->ttc_attr[i].zstd_dictid = (Oid) aopt->zstd_dictid;
+ ttc->ttc_attr[i].zstd_level = aopt->zstd_cmp_level;
+ }
+ }
if (ttc->ttc_oldvalues != NULL)
{
@@ -230,7 +243,7 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
Datum new_value;
ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
- new_value = toast_compress_datum(*value, attr->tai_compression);
+ new_value = toast_compress_datum(*value, attr->tai_compression, attr->zstd_dictid, attr->zstd_level);
if (DatumGetPointer(new_value) != NULL)
{
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index c090094ed0..282afbcef5 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -46,7 +46,8 @@ OBJS = \
pg_subscription.o \
pg_type.o \
storage.o \
- toasting.o
+ toasting.o \
+ pg_zstd_dictionaries.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index bd3554c0bf..493963b1b8 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -1071,7 +1071,9 @@ AddNewRelationType(const char *typeName,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ InvalidOid /* generate dictionary procedure - default */
+ );
}
/* --------------------------------
@@ -1394,7 +1396,9 @@ heap_create_with_catalog(const char *relname,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ InvalidOid /* generate dictionary procedure - default */
+ );
pfree(relarrayname);
}
diff --git a/src/backend/catalog/meson.build b/src/backend/catalog/meson.build
index 1958ea9238..8f0413189c 100644
--- a/src/backend/catalog/meson.build
+++ b/src/backend/catalog/meson.build
@@ -34,6 +34,7 @@ backend_sources += files(
'pg_type.c',
'storage.c',
'toasting.c',
+ 'pg_zstd_dictionaries.c',
)
diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c
index b36f81afb9..bbed8f64ad 100644
--- a/src/backend/catalog/pg_type.c
+++ b/src/backend/catalog/pg_type.c
@@ -120,6 +120,7 @@ TypeShellMake(const char *typeName, Oid typeNamespace, Oid ownerId)
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(-1);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(0);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(InvalidOid);
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(InvalidOid);
nulls[Anum_pg_type_typdefaultbin - 1] = true;
nulls[Anum_pg_type_typdefault - 1] = true;
nulls[Anum_pg_type_typacl - 1] = true;
@@ -223,7 +224,8 @@ TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims, /* Array dimensions for baseType */
bool typeNotNull,
- Oid typeCollation)
+ Oid typeCollation,
+ Oid generateDictionaryProcedure)
{
Relation pg_type_desc;
Oid typeObjectId;
@@ -378,6 +380,7 @@ TypeCreate(Oid newTypeOid,
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(typeMod);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(typNDims);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(typeCollation);
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(generateDictionaryProcedure);
/*
* initialize the default binary value for this type. Check for nulls of
@@ -679,6 +682,12 @@ GenerateTypeDependencies(HeapTuple typeTuple,
add_exact_object_address(&referenced, addrs_normal);
}
+ if (OidIsValid(typeForm->typebuildzstddictionary))
+ {
+ ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typebuildzstddictionary);
+ add_exact_object_address(&referenced, addrs_normal);
+ }
+
if (OidIsValid(typeForm->typsubscript))
{
ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typsubscript);
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
new file mode 100644
index 0000000000..f52f9decb9
--- /dev/null
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -0,0 +1,601 @@
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "access/heapam.h"
+#include "access/table.h"
+#include "access/relation.h"
+#include "access/tableam.h"
+#include "catalog/catalog.h"
+#include "catalog/dependency.h"
+#include "catalog/indexing.h"
+#include "catalog/pg_class_d.h"
+#include "catalog/pg_zstd_dictionaries.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+#include "catalog/pg_type.h"
+#include "catalog/namespace.h"
+#include "utils/builtins.h"
+#include "utils/fmgroids.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/hsearch.h"
+#include "access/toast_compression.h"
+#include "utils/attoptcache.h"
+#include "parser/analyze.h"
+#include "common/hashfn.h"
+#include "nodes/makefuncs.h"
+#include "access/reloptions.h"
+#include "miscadmin.h"
+#include "access/genam.h"
+#include "executor/tuptable.h"
+#include "access/htup_details.h"
+#include "access/sdir.h"
+#include "utils/lsyscache.h"
+#include "utils/relcache.h"
+#include "utils/memutils.h"
+#include "utils/varlena.h"
+#include "nodes/pg_list.h"
+
+#ifdef USE_ZSTD
+#include <zstd.h>
+#include <zdict.h>
+#endif
+
+#define TARG_ROWS 1000
+
+typedef struct SampleEntry SampleEntry;
+typedef struct SampleCollector SampleCollector;
+
+/* Structure to store a sample entry */
+struct SampleEntry
+{
+ void *data; /* Pointer to sample data */
+ size_t size; /* Size of the sample */
+};
+
+/* Structure to collect samples along with a hash table for deduplication */
+struct SampleCollector
+{
+ SampleEntry *samples; /* Dynamic array of pointers to SampleEntry */
+ int sample_count; /* Number of collected samples */
+};
+
+static bool build_zstd_dictionary_internal(Oid relid, AttrNumber attno);
+static Oid GetNewDictId(Relation relation, Oid indexId, AttrNumber dictIdColumn);
+
+/* ----------------------------------------------------------------
+ * Zstandard dictionary training related methods
+ * ----------------------------------------------------------------
+ */
+
+/*
+ * build_zstd_dictionary_internal
+ * 1) Validate that the given (relid, attno) can have a Zstd compression enabled on heap relation
+ * 2) Call the type-specific dictionary builder
+ * 3) Train a dictionary via ZDICT_trainFromBuffer()
+ * 4) Insert dictionary into pg_zstd_dictionaries
+ * 5) Update pg_attribute.attoptions with dictid
+ */
+pg_attribute_unused()
+static bool
+build_zstd_dictionary_internal(Oid relid, AttrNumber attno)
+{
+#ifdef USE_ZSTD
+ Relation catalogRel;
+ TupleDesc catTupDesc;
+ Oid dictid;
+ Relation rel;
+ TupleDesc tupleDesc;
+ Form_pg_attribute att;
+ AttributeOpts *attopt;
+ HeapTuple typeTup;
+ Form_pg_type typeForm;
+ Oid baseTypeOid;
+ Oid train_func;
+ Datum dictDatum;
+ ZstdTrainingData *dict;
+ char *samples_buffer;
+ size_t *sample_sizes;
+ int nitems;
+ uint32 dictionary_size;
+ void *dict_data;
+ size_t dict_size;
+
+ /* ----
+ * 1) Open user relation just to verify it's a normal table and has Zstd compression
+ * ----
+ */
+ rel = table_open(relid, AccessShareLock);
+ if (rel->rd_rel->relkind != RELKIND_RELATION)
+ {
+ table_close(rel, AccessShareLock);
+ return false; /* not a regular table */
+ }
+
+ /* If the column doesn't use Zstd, nothing to do */
+ tupleDesc = RelationGetDescr(rel);
+ att = TupleDescAttr(tupleDesc, attno - 1);
+ if (att->attcompression != TOAST_ZSTD_COMPRESSION)
+ {
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Check attoptions for user-requested dictionary size, etc. */
+ attopt = get_attribute_options(relid, attno);
+ if (attopt && attopt->zstd_dict_size == 0)
+ {
+ /* user explicitly says "no dictionary needed" */
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /*
+ * 2) Look up the type's custom dictionary builder function We'll call it
+ * to get sample data. Then we can close 'rel' because we don't need it
+ * open to do the actual Zdict training.
+ */
+ typeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typeTup))
+ {
+ table_close(rel, AccessShareLock);
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ }
+ typeForm = (Form_pg_type) GETSTRUCT(typeTup);
+
+ if (typeForm->typlen != -1)
+ {
+ ReleaseSysCache(typeTup);
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Get the base type */
+ baseTypeOid = get_element_type(typeForm->oid);
+ train_func = InvalidOid;
+
+ if (OidIsValid(baseTypeOid))
+ {
+ HeapTuple baseTypeTup;
+ Form_pg_type baseTypeForm;
+
+ /* It's an array type: get the base type's training function */
+ baseTypeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(baseTypeOid));
+ if (!HeapTupleIsValid(baseTypeTup))
+ ereport(ERROR,
+ (errmsg("Cache lookup failed for base type %u", baseTypeOid)));
+
+ baseTypeForm = (Form_pg_type) GETSTRUCT(baseTypeTup);
+ train_func = baseTypeForm->typebuildzstddictionary;
+ ReleaseSysCache(baseTypeTup);
+ }
+ else
+ train_func = typeForm->typebuildzstddictionary;
+
+ /* If the type does not supply a builder, skip */
+ if (!OidIsValid(train_func))
+ {
+ ReleaseSysCache(typeTup);
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Call the type-specific builder. It should return ZstdTrainingData */
+ dictDatum = OidFunctionCall2(train_func,
+ PointerGetDatum(rel), /* pass relation ref */
+ PointerGetDatum(att));
+ ReleaseSysCache(typeTup);
+
+ /* We no longer need the user relation open */
+ table_close(rel, AccessShareLock);
+
+ dict = (ZstdTrainingData *) DatumGetPointer(dictDatum);
+ if (!dict || dict->nitems == 0)
+ return false;
+
+ /*
+ * 3) Train a Zstd dictionary in-memory.
+ */
+ samples_buffer = dict->sample_buffer;
+ sample_sizes = dict->sample_sizes;
+ nitems = dict->nitems;
+
+ dictionary_size = (!attopt ? DEFAULT_ZSTD_DICT_SIZE
+ : attopt->zstd_dict_size);
+
+ /* Allocate buffer for dictionary training result */
+ dict_data = palloc(dictionary_size);
+ dict_size = ZDICT_trainFromBuffer(dict_data,
+ dictionary_size,
+ samples_buffer,
+ sample_sizes,
+ nitems);
+ if (ZDICT_isError(dict_size))
+ {
+ elog(LOG, "Zstd dictionary training failed: %s",
+ ZDICT_getErrorName(dict_size));
+ pfree(dict_data);
+ return false;
+ }
+
+ /*
+ * Finalize dictionary to embed a custom dictID. E.g. We can get a new Oid
+ * from pg_zstd_dictionaries here *before* we build the bytea. But for
+ * brevity, let's do it after opening pg_zstd_dictionaries (so we can do
+ * the dictionary insertion + ID assignment in one place).
+ *
+ * 4) Insert dictionary into pg_zstd_dictionaries We do that by opening
+ * the ZstdDictionariesRelation, generating a new dictid, forming a tuple,
+ * and inserting it.
+ *
+ * finalize to embed that 'dictid' in the dictionary itself
+ */
+ {
+ ZDICT_params_t fParams;
+ size_t final_dict_size;
+
+ /* Open the catalog relation with ShareRowExclusiveLock */
+ catalogRel = table_open(ZstdDictionariesRelationId, ShareRowExclusiveLock);
+ catTupDesc = RelationGetDescr(catalogRel);
+ dictid = GetNewDictId(catalogRel, ZstdDictidIndexId, Anum_pg_zstd_dictionaries_dictid);
+
+ memset(&fParams, 0, sizeof(fParams));
+ fParams.dictID = dictid; /* embed the newly allocated Oid as the
+ * dictID */
+
+ final_dict_size = ZDICT_finalizeDictionary(
+ dict_data, /* output buffer (reuse) */
+ dictionary_size, /* capacity */
+ dict_data, /* input dictionary from
+ * train step */
+ dict_size, /* size from train step */
+ samples_buffer,
+ sample_sizes,
+ nitems,
+ fParams);
+
+ /* Verify that the embedded dictionary ID matches the expected value */
+ if (dictid != (Oid) ZDICT_getDictID(dict_data, final_dict_size))
+ elog(ERROR, "Zstd dictionary ID mismatch");
+
+ if (ZDICT_isError(final_dict_size))
+ {
+ elog(LOG, "Zstd dictionary finalization failed: %s",
+ ZDICT_getErrorName(final_dict_size));
+ pfree(dict_data);
+ table_close(catalogRel, ShareRowExclusiveLock);
+ return false;
+ }
+
+ /* Now copy that finalized dictionary into a bytea. */
+ {
+ /* We’ll store this bytea in pg_zstd_dictionaries. */
+ Datum values[Natts_pg_zstd_dictionaries];
+ bool nulls[Natts_pg_zstd_dictionaries];
+ HeapTuple tup;
+
+ bytea *dict_bytea = (bytea *) palloc(VARHDRSZ + final_dict_size);
+
+ SET_VARSIZE(dict_bytea, VARHDRSZ + final_dict_size);
+ memcpy(VARDATA(dict_bytea), dict_data, final_dict_size);
+
+ MemSet(values, 0, sizeof(values));
+ MemSet(nulls, false, sizeof(nulls));
+
+ values[Anum_pg_zstd_dictionaries_dictid - 1] = ObjectIdGetDatum(dictid);
+ values[Anum_pg_zstd_dictionaries_dict - 1] = PointerGetDatum(dict_bytea);
+
+ tup = heap_form_tuple(catTupDesc, values, nulls);
+ CatalogTupleInsert(catalogRel, tup);
+ heap_freetuple(tup);
+
+ pfree(dict_bytea);
+ }
+
+ pfree(dict_data);
+ }
+
+ pfree(samples_buffer);
+ pfree(sample_sizes);
+ pfree(dict);
+
+ /*
+ * 5) Update pg_attribute.attoptions with "zstd_dictid" => dictid so the
+ * column knows which dictionary to use at compression time.
+ */
+ {
+ Relation attRel = table_open(AttributeRelationId, RowExclusiveLock);
+ HeapTuple atttup,
+ newtuple;
+ Datum attoptionsDatum,
+ newOptions;
+ bool isnull;
+ Datum repl_val[Natts_pg_attribute];
+ bool repl_null[Natts_pg_attribute];
+ bool repl_repl[Natts_pg_attribute];
+ DefElem *def;
+
+ atttup = SearchSysCacheAttNum(relid, attno);
+ if (!HeapTupleIsValid(atttup))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_COLUMN),
+ errmsg("column number %d of relation \"%u\" does not exist",
+ attno, relid)));
+
+ /* Build new attoptions with zstd_dictid=... */
+ def = makeDefElem("zstd_dictid",
+ (Node *) makeString(psprintf("%u", dictid)),
+ -1);
+
+ attoptionsDatum = SysCacheGetAttr(ATTNUM, atttup,
+ Anum_pg_attribute_attoptions,
+ &isnull);
+ newOptions = transformRelOptions(isnull ? (Datum) 0 : attoptionsDatum,
+ list_make1(def),
+ NULL, NULL,
+ false, false);
+ /* Validate them (throws error if invalid) */
+ (void) attribute_reloptions(newOptions, true);
+
+ MemSet(repl_null, false, sizeof(repl_null));
+ MemSet(repl_repl, false, sizeof(repl_repl));
+
+ if (newOptions != (Datum) 0)
+ repl_val[Anum_pg_attribute_attoptions - 1] = newOptions;
+ else
+ repl_null[Anum_pg_attribute_attoptions - 1] = true;
+
+ repl_repl[Anum_pg_attribute_attoptions - 1] = true;
+
+ newtuple = heap_modify_tuple(atttup,
+ RelationGetDescr(attRel),
+ repl_val,
+ repl_null,
+ repl_repl);
+
+ CatalogTupleUpdate(attRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+
+ ReleaseSysCache(atttup);
+
+ table_close(attRel, NoLock);
+ }
+
+ /**
+ * Done inserting dictionary and updating attribute.
+ * Unlock the table (locks remain held until transaction commit)
+ */
+ table_close(catalogRel, NoLock);
+
+ return true;
+#else
+ return false;
+#endif
+}
+
+/*
+ * Acquire a new unique DictId for a relation.
+ *
+ * Assumes the relation is already locked with ShareRowExclusiveLock,
+ * ensuring that concurrent transactions cannot generate duplicate DictIds.
+ */
+pg_attribute_unused()
+static Oid
+GetNewDictId(Relation relation, Oid indexId, AttrNumber dictIdColumn)
+{
+ Relation indexRel = index_open(indexId, AccessShareLock);
+ Oid maxDictId = InvalidDictId;
+ SysScanDesc scan;
+ HeapTuple tuple;
+ bool collision;
+ ScanKeyData key;
+ Oid newDictId;
+
+ /* Retrieve the maximum existing DictId by scanning in reverse order */
+ scan = systable_beginscan_ordered(relation, indexRel, SnapshotAny, 0, NULL);
+ tuple = systable_getnext_ordered(scan, BackwardScanDirection);
+ if (HeapTupleIsValid(tuple))
+ {
+ Datum value;
+ bool isNull;
+
+ value = heap_getattr(tuple, dictIdColumn, RelationGetDescr(relation), &isNull);
+ if (!isNull)
+ maxDictId = DatumGetObjectId(value);
+ }
+ systable_endscan(scan);
+
+ newDictId = maxDictId + 1;
+ Assert(newDictId != InvalidDictId);
+
+ /* Check that the new DictId is indeed unique */
+ ScanKeyInit(&key,
+ dictIdColumn,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(newDictId));
+
+ scan = systable_beginscan(relation, indexRel->rd_id, true,
+ SnapshotAny, 1, &key);
+ collision = HeapTupleIsValid(systable_getnext(scan));
+ systable_endscan(scan);
+
+ if (collision)
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("unexpected collision for new DictId %d", newDictId)));
+
+ return newDictId;
+}
+
+/*
+ * get_zstd_dict - Fetches the ZSTD dictionary from the catalog
+ *
+ * dictid: The Oid of the dictionary to fetch.
+ *
+ * Returns: A pointer to a bytea containing the dictionary data.
+ */
+bytea *
+get_zstd_dict(Oid dictid)
+{
+ HeapTuple tuple;
+ Datum datum;
+ bool isNull;
+ bytea *dict_bytea;
+ bytea *result;
+ Size bytea_len;
+
+ /* Fetch the dictionary tuple from the syscache */
+ tuple = SearchSysCache1(ZSTDDICTIDOID, ObjectIdGetDatum(dictid));
+ if (!HeapTupleIsValid(tuple))
+ ereport(ERROR, (errmsg("Cache lookup failed for dictid %u", dictid)));
+
+ /* Get the dictionary attribute from the tuple */
+ datum = SysCacheGetAttr(ATTNUM, tuple, Anum_pg_zstd_dictionaries_dict, &isNull);
+ if (isNull)
+ ereport(ERROR, (errmsg("Dictionary not found for dictid %u", dictid)));
+
+ dict_bytea = DatumGetByteaP(datum);
+ if (dict_bytea == NULL)
+ ereport(ERROR, (errmsg("Failed to fetch dictionary")));
+
+ /* Determine the total size of the bytea (header + data) */
+ bytea_len = VARSIZE(dict_bytea);
+
+ result = palloc(bytea_len);
+ memcpy(result, dict_bytea, bytea_len);
+
+ /* Release the syscache tuple; the returned bytea is now independent */
+ ReleaseSysCache(tuple);
+
+ return result;
+}
+
+/*
+ * zstd_dictionary_builder
+ * Acquire samples from a column, store them in a SampleCollector,
+ * filter them, then build a ZstdTrainingData struct.
+ */
+Datum
+zstd_dictionary_builder(PG_FUNCTION_ARGS)
+{
+ ZstdTrainingData *dict = palloc0(sizeof(ZstdTrainingData));
+ Relation rel = (Relation) PG_GETARG_POINTER(0);
+ Form_pg_attribute att = (Form_pg_attribute) PG_GETARG_POINTER(1);
+ TupleDesc tupleDesc = RelationGetDescr(rel);
+
+ /* Acquire up to TARG_ROWS sample rows. */
+ HeapTuple *sample_rows = palloc(TARG_ROWS * sizeof(HeapTuple));
+ double totalrows = 0,
+ totaldeadrows = 0;
+ int num_sampled = acquire_sample_rows(rel, 0, sample_rows,
+ TARG_ROWS,
+ &totalrows,
+ &totaldeadrows);
+
+ /* Create a collector to accumulate raw varlena samples. */
+ size_t filtered_sample_count = 0;
+ size_t filtered_samples_size = 0;
+ char *samples_buffer;
+ size_t *sample_sizes;
+ size_t current_offset = 0;
+
+ SampleCollector *collector = palloc(sizeof(SampleCollector));
+
+ collector->samples = palloc(num_sampled * sizeof(SampleEntry));
+ collector->sample_count = 0;
+
+ /* Extract column data from each sampled row. */
+ for (int i = 0; i < num_sampled; i++)
+ {
+ bool isnull;
+ Datum value;
+
+ CHECK_FOR_INTERRUPTS();
+
+ value = heap_getattr(sample_rows[i],
+ att->attnum,
+ tupleDesc,
+ &isnull);
+ if (!isnull)
+ {
+ struct varlena *attr;
+ size_t size;
+ void *data;
+ SampleEntry entry;
+ int idx;
+
+ attr = (struct varlena *) PG_DETOAST_DATUM(value);
+ size = VARSIZE_ANY_EXHDR(attr);
+
+ if (filtered_samples_size + size > MaxAllocSize)
+ break;
+
+ data = palloc(size);
+ memcpy(data, VARDATA_ANY(attr), size);
+
+ entry.data = data;
+ entry.size = size;
+
+ idx = collector->sample_count;
+ collector->samples[idx] = entry;
+ collector->sample_count++;
+
+ filtered_samples_size += size;
+ filtered_sample_count++;
+ }
+ }
+
+ if (filtered_sample_count == 0)
+ {
+ /* No samples were collected, or they were too large. */
+ PG_RETURN_POINTER(dict);
+ }
+
+ /* Allocate a buffer for all sample data, plus an array of sample sizes. */
+ samples_buffer = palloc(filtered_samples_size);
+ sample_sizes = palloc(filtered_sample_count * sizeof(size_t));
+
+ /*
+ * Concatenate the samples into samples_buffer, recording each sample's
+ * size in sample_sizes.
+ */
+ current_offset = 0;
+ for (int i = 0; i < filtered_sample_count; i++)
+ {
+ memcpy(samples_buffer + current_offset,
+ collector->samples[i].data,
+ collector->samples[i].size);
+
+ sample_sizes[i] = collector->samples[i].size;
+ current_offset += collector->samples[i].size;
+
+ pfree(collector->samples[i].data);
+ }
+ pfree(collector->samples);
+ pfree(collector);
+
+ dict->sample_buffer = samples_buffer;
+ dict->sample_sizes = sample_sizes;
+ dict->nitems = filtered_sample_count;
+
+ PG_RETURN_POINTER(dict);
+}
+
+Datum
+build_zstd_dict_for_attribute(PG_FUNCTION_ARGS)
+{
+#ifndef USE_ZSTD
+ PG_RETURN_BOOL(false);
+#else
+ text *tablename = PG_GETARG_TEXT_PP(0);
+ RangeVar *tablerel;
+ Oid tableoid = InvalidOid;
+ AttrNumber attno = PG_GETARG_INT32(1);
+ bool success;
+
+ /* Look up table name. */
+ tablerel = makeRangeVarFromNameList(textToQualifiedNameList(tablename));
+ tableoid = RangeVarGetRelid(tablerel, NoLock, false);
+ success = build_zstd_dictionary_internal(tableoid, attno);
+ PG_RETURN_BOOL(success);
+#endif
+}
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 2b5fbdcbd8..2b5500f45f 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -55,7 +55,7 @@
#include "utils/sortsupport.h"
#include "utils/syscache.h"
#include "utils/timestamp.h"
-
+#include "parser/analyze.h"
/* Per-index data for ANALYZE */
typedef struct AnlIndexData
@@ -85,9 +85,6 @@ static void compute_index_stats(Relation onerel, double totalrows,
MemoryContext col_context);
static VacAttrStats *examine_attribute(Relation onerel, int attnum,
Node *index_expr);
-static int acquire_sample_rows(Relation onerel, int elevel,
- HeapTuple *rows, int targrows,
- double *totalrows, double *totaldeadrows);
static int compare_rows(const void *a, const void *b, void *arg);
static int acquire_inherited_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
@@ -1195,7 +1192,7 @@ block_sampling_read_stream_next(ReadStream *stream,
* block. The previous sampling method put too much credence in the row
* density near the start of the table.
*/
-static int
+int
acquire_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
double *totalrows, double *totaldeadrows)
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index 3cb3ca1cca..c583e48167 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -95,6 +95,7 @@ typedef struct
bool updateTypmodout;
bool updateAnalyze;
bool updateSubscript;
+ bool updateGenerateDictionary;
/* New values for relevant attributes */
char storage;
Oid receiveOid;
@@ -103,6 +104,7 @@ typedef struct
Oid typmodoutOid;
Oid analyzeOid;
Oid subscriptOid;
+ Oid buildZstdDictionary;
} AlterTypeRecurseParams;
/* Potentially set by pg_upgrade_support functions */
@@ -122,6 +124,7 @@ static Oid findTypeSendFunction(List *procname, Oid typeOid);
static Oid findTypeTypmodinFunction(List *procname);
static Oid findTypeTypmodoutFunction(List *procname);
static Oid findTypeAnalyzeFunction(List *procname, Oid typeOid);
+static Oid findTypeGenerateDictionaryFunction(List *procname, Oid typeOid);
static Oid findTypeSubscriptingFunction(List *procname, Oid typeOid);
static Oid findRangeSubOpclass(List *opcname, Oid subtype);
static Oid findRangeCanonicalFunction(List *procname, Oid typeOid);
@@ -162,6 +165,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
List *typmodoutName = NIL;
List *analyzeName = NIL;
List *subscriptName = NIL;
+ List *generateDictionaryName = NIL;
char category = TYPCATEGORY_USER;
bool preferred = false;
char delimiter = DEFAULT_TYPDELIM;
@@ -190,6 +194,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
DefElem *alignmentEl = NULL;
DefElem *storageEl = NULL;
DefElem *collatableEl = NULL;
+ DefElem *generateDictionaryEl = NULL;
Oid inputOid;
Oid outputOid;
Oid receiveOid = InvalidOid;
@@ -198,6 +203,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
Oid typmodoutOid = InvalidOid;
Oid analyzeOid = InvalidOid;
Oid subscriptOid = InvalidOid;
+ Oid buildZstdDictionary = InvalidOid;
char *array_type;
Oid array_oid;
Oid typoid;
@@ -323,6 +329,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
defelp = &storageEl;
else if (strcmp(defel->defname, "collatable") == 0)
defelp = &collatableEl;
+ else if (strcmp(defel->defname, "build_zstd_dict") == 0)
+ defelp = &generateDictionaryEl;
else
{
/* WARNING, not ERROR, for historical backwards-compatibility */
@@ -455,6 +463,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
}
if (collatableEl)
collation = defGetBoolean(collatableEl) ? DEFAULT_COLLATION_OID : InvalidOid;
+ if (generateDictionaryEl)
+ generateDictionaryName = defGetQualifiedName(generateDictionaryEl);
/*
* make sure we have our required definitions
@@ -516,6 +526,15 @@ DefineType(ParseState *pstate, List *names, List *parameters)
errmsg("element type cannot be specified without a subscripting function")));
}
+ if (generateDictionaryName)
+ {
+ if (internalLength != -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type build_zstd_dict function must be specified only if data type is variable length.")));
+ buildZstdDictionary = findTypeGenerateDictionaryFunction(generateDictionaryName, typoid);
+ }
+
/*
* Check permissions on functions. We choose to require the creator/owner
* of a type to also own the underlying functions. Since creating a type
@@ -550,6 +569,9 @@ DefineType(ParseState *pstate, List *names, List *parameters)
if (analyzeOid && !object_ownercheck(ProcedureRelationId, analyzeOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(analyzeName));
+ if (buildZstdDictionary && !object_ownercheck(ProcedureRelationId, buildZstdDictionary, GetUserId()))
+ aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
+ NameListToString(generateDictionaryName));
if (subscriptOid && !object_ownercheck(ProcedureRelationId, subscriptOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(subscriptName));
@@ -601,7 +623,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array Dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ buildZstdDictionary); /* build_zstd_dict procedure */
Assert(typoid == address.objectId);
/*
@@ -643,7 +666,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ InvalidOid); /* build_zstd_dict procedure */
pfree(array_type);
@@ -706,6 +730,7 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
Oid receiveProcedure;
Oid sendProcedure;
Oid analyzeProcedure;
+ Oid buildZstdDictionary;
bool byValue;
char category;
char delimiter;
@@ -842,6 +867,9 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
/* Analysis function */
analyzeProcedure = baseType->typanalyze;
+ /* Generate dictionary function */
+ buildZstdDictionary = baseType->typebuildzstddictionary;
+
/*
* Domains don't need a subscript function, since they are not
* subscriptable on their own. If the base type is subscriptable, the
@@ -1078,7 +1106,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
basetypeMod, /* typeMod value */
typNDims, /* Array dimensions for base type */
typNotNull, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ buildZstdDictionary); /* build_zstd_dict procedure */
/*
* Create the array type that goes with it.
@@ -1119,7 +1148,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ InvalidOid); /* build_zstd_dict procedure */
pfree(domainArrayName);
@@ -1241,7 +1271,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
/* Enter the enum's values into pg_enum */
EnumValuesCreate(enumTypeAddr.objectId, stmt->vals);
@@ -1282,7 +1313,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
pfree(enumArrayName);
@@ -1583,7 +1615,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ InvalidOid); /* generate dictionary procedure - default */
Assert(typoid == InvalidOid || typoid == address.objectId);
typoid = address.objectId;
@@ -1650,7 +1683,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ InvalidOid); /* generate dictionary procedure - default */
Assert(multirangeOid == mltrngaddress.objectId);
/* Create the entry in pg_range */
@@ -1693,7 +1727,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ InvalidOid); /* generate dictionary procedure - default */
pfree(rangeArrayName);
@@ -1732,7 +1767,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ InvalidOid); /* generate dictionary procedure - default */
/* And create the constructor functions for this range type */
makeRangeConstructors(typeName, typeNamespace, typoid, rangeSubtype);
@@ -2257,6 +2293,31 @@ findTypeAnalyzeFunction(List *procname, Oid typeOid)
return procOid;
}
+static Oid
+findTypeGenerateDictionaryFunction(List *procname, Oid typeOid)
+{
+ Oid argList[2];
+ Oid procOid;
+
+ argList[0] = OIDOID;
+ argList[1] = INT4OID;
+
+ procOid = LookupFuncName(procname, 2, argList, true);
+ if (!OidIsValid(procOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_FUNCTION),
+ errmsg("function %s does not exist",
+ func_signature_string(procname, 1, NIL, argList))));
+
+ if (get_func_rettype(procOid) != INTERNALOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type build zstd dictionary function %s must return type %s",
+ NameListToString(procname), "internal")));
+
+ return procOid;
+}
+
static Oid
findTypeSubscriptingFunction(List *procname, Oid typeOid)
{
@@ -4440,6 +4501,19 @@ AlterType(AlterTypeStmt *stmt)
/* Replacing a subscript function requires superuser. */
requireSuper = true;
}
+ else if (strcmp(defel->defname, "build_zstd_dict") == 0)
+ {
+ if (defel->arg != NULL)
+ atparams.buildZstdDictionary =
+ findTypeGenerateDictionaryFunction(defGetQualifiedName(defel),
+ typeOid);
+ else
+ atparams.buildZstdDictionary = InvalidOid; /* NONE, remove function */
+
+ atparams.updateGenerateDictionary = true;
+ /* Replacing a canonical function requires superuser. */
+ requireSuper = true;
+ }
/*
* The rest of the options that CREATE accepts cannot be changed.
@@ -4602,6 +4676,11 @@ AlterTypeRecurse(Oid typeOid, bool isImplicitArray,
replaces[Anum_pg_type_typsubscript - 1] = true;
values[Anum_pg_type_typsubscript - 1] = ObjectIdGetDatum(atparams->subscriptOid);
}
+ if (atparams->updateGenerateDictionary)
+ {
+ replaces[Anum_pg_type_typebuildzstddictionary - 1] = true;
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(atparams->buildZstdDictionary);
+ }
newtup = heap_modify_tuple(tup, RelationGetDescr(catalog),
values, nulls, replaces);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index e455657170..40e29452c8 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5184,6 +5184,9 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_ZSTD_COMPRESSION_ID:
+ result = "zstd";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index ad25cbb39c..e03ac8dddc 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -453,6 +453,9 @@ static const struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd", TOAST_ZSTD_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2d1de9c37b..47773e2919 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -731,7 +731,7 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate
#row_security = on
#default_table_access_method = 'heap'
#default_tablespace = '' # a tablespace name, '' uses the default
-#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
+#default_toast_compression = 'pglz' # 'pglz' or 'lz4' or 'zstd'
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#check_function_bodies = on
diff --git a/src/bin/pg_amcheck/t/004_verify_heapam.pl b/src/bin/pg_amcheck/t/004_verify_heapam.pl
index 2a3af2666f..3030998e58 100644
--- a/src/bin/pg_amcheck/t/004_verify_heapam.pl
+++ b/src/bin/pg_amcheck/t/004_verify_heapam.pl
@@ -75,7 +75,7 @@ use Test::More;
# xx t_bits: x offset = 23 C
# xx xx xx xx xx xx xx xx 'a': xxxxxxxx offset = 24 LL
# xx xx xx xx xx xx xx xx 'b': xxxxxxxx offset = 32 CCCCCCCC
-# xx xx xx xx xx xx xx xx 'c': xxxxxxxx offset = 40 CCllLL
+# xx xx xx xx xx xx xx xx 'c': xxxxxxxx offset = 40 CCllLLL
# xx xx xx xx xx xx xx xx : xxxxxxxx ...continued
# xx xx : xx ...continued
#
@@ -83,8 +83,8 @@ use Test::More;
# it is convenient enough to do it this way. We define packing code
# constants here, where they can be compared easily against the layout.
-use constant HEAPTUPLE_PACK_CODE => 'LLLSSSSSCCLLCCCCCCCCCCllLL';
-use constant HEAPTUPLE_PACK_LENGTH => 58; # Total size
+use constant HEAPTUPLE_PACK_CODE => 'LLLSSSSSCCLLCCCCCCCCCCllLLL';
+use constant HEAPTUPLE_PACK_LENGTH => 62; # Total size
# Read a tuple of our table from a heap page.
#
@@ -130,7 +130,8 @@ sub read_tuple
c_va_rawsize => shift,
c_va_extinfo => shift,
c_va_valueid => shift,
- c_va_toastrelid => shift);
+ c_va_toastrelid => shift,
+ c_va_cmp_alg => shift);
# Stitch together the text for column 'b'
$tup{b} = join('', map { chr($tup{"b_body$_"}) } (1 .. 7));
return \%tup;
@@ -163,7 +164,8 @@ sub write_tuple
$tup->{b_body6}, $tup->{b_body7},
$tup->{c_va_header}, $tup->{c_va_vartag},
$tup->{c_va_rawsize}, $tup->{c_va_extinfo},
- $tup->{c_va_valueid}, $tup->{c_va_toastrelid});
+ $tup->{c_va_valueid}, $tup->{c_va_toastrelid},
+ $tup->{c_va_cmp_alg});
sysseek($fh, $offset, 0)
or BAIL_OUT("sysseek failed: $!");
defined(syswrite($fh, $buffer, HEAPTUPLE_PACK_LENGTH))
@@ -496,7 +498,7 @@ for (my $tupidx = 0; $tupidx < $ROWCOUNT; $tupidx++)
$tup->{t_hoff} += 128;
push @expected,
- qr/${$header}data begins at offset 152 beyond the tuple length 58/,
+ qr/${$header}data begins at offset 152 beyond the tuple length 62/,
qr/${$header}tuple data should begin at byte 24, but actually begins at byte 152 \(3 attributes, no nulls\)/;
}
elsif ($offnum == 6)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 4f4ad2ee15..02bec765ad 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -8965,7 +8965,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
"a.attalign,\n"
"a.attislocal,\n"
"pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n"
- "array_to_string(a.attoptions, ', ') AS attoptions,\n"
+ "array_to_string(ARRAY(SELECT x FROM unnest(a.attoptions) AS x \n"
+ "WHERE x NOT LIKE 'zstd_dictid=%'), ', ') AS attoptions, \n"
"CASE WHEN a.attcollation <> t.typcollation "
"THEN a.attcollation ELSE 0 END AS attcollation,\n"
"pg_catalog.array_to_string(ARRAY("
@@ -11784,12 +11785,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
char *typmodout;
char *typanalyze;
char *typsubscript;
+ char *typebuildzstddictionary;
Oid typreceiveoid;
Oid typsendoid;
Oid typmodinoid;
Oid typmodoutoid;
Oid typanalyzeoid;
Oid typsubscriptoid;
+ Oid typebuildzstddictionaryoid;
char *typcategory;
char *typispreferred;
char *typdelim;
@@ -11822,10 +11825,18 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query,
"typsubscript, "
- "typsubscript::pg_catalog.oid AS typsubscriptoid ");
+ "typsubscript::pg_catalog.oid AS typsubscriptoid, ");
else
appendPQExpBufferStr(query,
- "'-' AS typsubscript, 0 AS typsubscriptoid ");
+ "'-' AS typsubscript, 0 AS typsubscriptoid, ");
+
+ if (fout->remoteVersion >= 180000)
+ appendPQExpBufferStr(query,
+ "typebuildzstddictionary, "
+ "typebuildzstddictionary::pg_catalog.oid AS typebuildzstddictionaryoid ");
+ else
+ appendPQExpBufferStr(query,
+ "'-' AS typebuildzstddictionary, 0 AS typebuildzstddictionaryoid ");
appendPQExpBufferStr(query, "FROM pg_catalog.pg_type "
"WHERE oid = $1");
@@ -11850,12 +11861,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
typmodout = PQgetvalue(res, 0, PQfnumber(res, "typmodout"));
typanalyze = PQgetvalue(res, 0, PQfnumber(res, "typanalyze"));
typsubscript = PQgetvalue(res, 0, PQfnumber(res, "typsubscript"));
+ typebuildzstddictionary = PQgetvalue(res, 0, PQfnumber(res, "typebuildzstddictionary"));
typreceiveoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typreceiveoid")));
typsendoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsendoid")));
typmodinoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodinoid")));
typmodoutoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodoutoid")));
typanalyzeoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typanalyzeoid")));
typsubscriptoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsubscriptoid")));
+ typebuildzstddictionaryoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typebuildzstddictionaryoid")));
typcategory = PQgetvalue(res, 0, PQfnumber(res, "typcategory"));
typispreferred = PQgetvalue(res, 0, PQfnumber(res, "typispreferred"));
typdelim = PQgetvalue(res, 0, PQfnumber(res, "typdelim"));
@@ -11911,7 +11924,8 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout);
if (OidIsValid(typanalyzeoid))
appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze);
-
+ if (OidIsValid(typebuildzstddictionaryoid))
+ appendPQExpBuffer(q, ",\n BUILD_ZSTD_DICT = %s", typebuildzstddictionary);
if (strcmp(typcollatable, "t") == 0)
appendPQExpBufferStr(q, ",\n COLLATABLE = true");
@@ -17170,6 +17184,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
case 'l':
cmname = "lz4";
break;
+ case 'z':
+ cmname = "zstd";
+ break;
default:
cmname = NULL;
break;
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index e6cf468ac9..0ba37bb175 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -2167,8 +2167,9 @@ describeOneTableDetails(const char *schemaname,
/* these strings are literal in our syntax, so not translated. */
printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
(compression[0] == 'l' ? "lz4" :
- (compression[0] == '\0' ? "" :
- "???"))),
+ (compression[0] == 'z' ? "zstd" :
+ (compression[0] == '\0' ? "" :
+ "???")))),
false, false);
}
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 13c4612cee..afff1f80e5 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -38,7 +38,8 @@ typedef enum ToastCompressionId
{
TOAST_PGLZ_COMPRESSION_ID = 0,
TOAST_LZ4_COMPRESSION_ID = 1,
- TOAST_INVALID_COMPRESSION_ID = 2,
+ TOAST_ZSTD_COMPRESSION_ID = 2,
+ TOAST_INVALID_COMPRESSION_ID = 3
} ToastCompressionId;
/*
@@ -48,10 +49,15 @@ typedef enum ToastCompressionId
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_COMPRESSION 'z'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
+#define InvalidDictId 0
+#define DEFAULT_ZSTD_CMP_LEVEL 3 /* Reffered from
+ * ZSTD_CLEVEL_DEFAULT */
+#define DEFAULT_ZSTD_DICT_SIZE (4 * 1024) /* 4 KB */
/* pglz compression/decompression routines */
extern struct varlena *pglz_compress_datum(const struct varlena *value);
@@ -65,6 +71,11 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* zstd compression/decompression routines */
+extern struct varlena *zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index e6ab8afffb..5df84e5e80 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -33,6 +33,8 @@ typedef struct
int32 tai_size;
uint8 tai_colflags;
char tai_compression;
+ Oid zstd_dictid;
+ int zstd_level;
} ToastAttrInfo;
/*
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index 06ae8583c1..26528850ba 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -25,6 +25,7 @@ typedef struct toast_compress_header
int32 vl_len_; /* varlena header (do not touch directly!) */
uint32 tcinfo; /* 2 bits for compression method and 30 bits
* external size; see va_extinfo */
+ uint32 ext_alg;
} toast_compress_header;
/*
@@ -33,19 +34,31 @@ typedef struct toast_compress_header
*/
#define TOAST_COMPRESS_EXTSIZE(ptr) \
(((toast_compress_header *) (ptr))->tcinfo & VARLENA_EXTSIZE_MASK)
-#define TOAST_COMPRESS_METHOD(ptr) \
- (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
-
-#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
- do { \
- Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
- Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_compress_header *) (ptr))->tcinfo = \
- (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+#define TOAST_COMPRESS_METHOD(PTR) \
+ ( ((((toast_compress_header *) (PTR))->tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((toast_compress_header *) (PTR))->ext_alg) \
+ : ( (((toast_compress_header *) (PTR))->tcinfo) >> VARLENA_EXTSIZE_BITS ) )
+
+#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
+ do { \
+ Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
+ Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_ZSTD_COMPRESSION_ID); \
+ /* If the compression method is less than TOAST_ZSTD_COMPRESSION_ID, don't use ext_alg */ \
+ if ((cm_method) < TOAST_ZSTD_COMPRESSION_ID) { \
+ ((toast_compress_header *) (ptr))->tcinfo = \
+ (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+ } else { \
+ /* For compression methods after lz4, use 'VARLENA_EXTENDED_COMPRESSION_FLAG' \
+ in the top bits of tcinfo to indicate compression algorithm is stored in ext_alg. */ \
+ ((toast_compress_header *) (ptr))->tcinfo = \
+ (len) | ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG << VARLENA_EXTSIZE_BITS); \
+ ((toast_compress_header *) (ptr))->ext_alg = (cm_method); \
+ } \
} while (0)
-extern Datum toast_compress_datum(Datum value, char cmethod);
+extern Datum toast_compress_datum(Datum value, char cmethod, Oid zstd_dictid, int zstd_level);
extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile
index 2bbc7805fe..1ecd76dd31 100644
--- a/src/include/catalog/Makefile
+++ b/src/include/catalog/Makefile
@@ -81,7 +81,8 @@ CATALOG_HEADERS := \
pg_publication_namespace.h \
pg_publication_rel.h \
pg_subscription.h \
- pg_subscription_rel.h
+ pg_subscription_rel.h \
+ pg_zstd_dictionaries.h
GENERATED_HEADERS := $(CATALOG_HEADERS:%.h=%_d.h)
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index f0962e17b3..d8a14432bd 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 202503031
+#define CATALOG_VERSION_NO 202503051
#endif
diff --git a/src/include/catalog/meson.build b/src/include/catalog/meson.build
index ec1cf467f6..e9cb6d911c 100644
--- a/src/include/catalog/meson.build
+++ b/src/include/catalog/meson.build
@@ -69,6 +69,7 @@ catalog_headers = [
'pg_publication_rel.h',
'pg_subscription.h',
'pg_subscription_rel.h',
+ 'pg_zstd_dictionaries.h',
]
# The .dat files we need can just be listed alphabetically.
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 134b3dd868..376375c055 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -12450,4 +12450,14 @@
proargtypes => 'int4',
prosrc => 'gist_stratnum_common' },
+# ZSTD generate dictionary training functions
+{ oid => '9241', descr => 'ZSTD generate dictionary support',
+ proname => 'zstd_dictionary_builder', prorettype => 'internal',
+ proargtypes => 'internal internal',
+ prosrc => 'zstd_dictionary_builder' },
+
+{ oid => '9242', descr => 'Build zstd dictionaries for a column.',
+ proname => 'build_zstd_dict_for_attribute', prorettype => 'bool',
+ proargtypes => 'text int4',
+ prosrc => 'build_zstd_dict_for_attribute' },
]
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index 6dca77e0a2..58a389a78c 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -40,7 +40,7 @@
descr => 'variable-length string, binary values escaped',
typname => 'bytea', typlen => '-1', typbyval => 'f', typcategory => 'U',
typinput => 'byteain', typoutput => 'byteaout', typreceive => 'bytearecv',
- typsend => 'byteasend', typalign => 'i', typstorage => 'x' },
+ typsend => 'byteasend', typalign => 'i', typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder' },
{ oid => '18', array_type_oid => '1002', descr => 'single character',
typname => 'char', typlen => '1', typbyval => 't', typcategory => 'Z',
typinput => 'charin', typoutput => 'charout', typreceive => 'charrecv',
@@ -83,7 +83,7 @@
typname => 'text', typlen => '-1', typbyval => 'f', typcategory => 'S',
typispreferred => 't', typinput => 'textin', typoutput => 'textout',
typreceive => 'textrecv', typsend => 'textsend', typalign => 'i',
- typstorage => 'x', typcollation => 'default' },
+ typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder', typcollation => 'default' },
{ oid => '26', array_type_oid => '1028',
descr => 'object identifier(oid), maximum 4 billion',
typname => 'oid', typlen => '4', typbyval => 't', typcategory => 'N',
@@ -446,7 +446,7 @@
typname => 'jsonb', typlen => '-1', typbyval => 'f', typcategory => 'U',
typsubscript => 'jsonb_subscript_handler', typinput => 'jsonb_in',
typoutput => 'jsonb_out', typreceive => 'jsonb_recv', typsend => 'jsonb_send',
- typalign => 'i', typstorage => 'x' },
+ typalign => 'i', typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder' },
{ oid => '4072', array_type_oid => '4073', descr => 'JSON path',
typname => 'jsonpath', typlen => '-1', typbyval => 'f', typcategory => 'U',
typinput => 'jsonpath_in', typoutput => 'jsonpath_out',
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index ff666711a5..bd82da8a88 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -227,6 +227,11 @@ CATALOG(pg_type,1247,TypeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(71,TypeRelati
*/
Oid typcollation BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_collation);
+ /*
+ * Custom generate dictionary procedure for the datatype (0 selects the
+ * default).
+ */
+ regproc typebuildzstddictionary BKI_DEFAULT(-) BKI_ARRAY_DEFAULT(-) BKI_LOOKUP_OPT(pg_proc);
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/*
@@ -380,7 +385,8 @@ extern ObjectAddress TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims,
bool typeNotNull,
- Oid typeCollation);
+ Oid typeCollation,
+ Oid generateDictionaryProcedure);
extern void GenerateTypeDependencies(HeapTuple typeTuple,
Relation typeCatalog,
diff --git a/src/include/catalog/pg_zstd_dictionaries.h b/src/include/catalog/pg_zstd_dictionaries.h
new file mode 100644
index 0000000000..b200ab541e
--- /dev/null
+++ b/src/include/catalog/pg_zstd_dictionaries.h
@@ -0,0 +1,53 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_zstd_dictionaries.h
+ * definition of the "zstd dictionay" system catalog (pg_zstd_dictionaries)
+ *
+ * src/include/catalog/pg_zstd_dictionaries.h
+ *
+ * NOTES
+ * The Catalog.pm module reads this file and derives schema
+ * information.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_ZSTD_DICTIONARIES_H
+#define PG_ZSTD_DICTIONARIES_H
+
+#include "catalog/genbki.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+
+/* ----------------
+ * pg_zstd_dictionaries definition. cpp turns this into
+ * typedef struct FormData_pg_zstd_dictionaries
+ * ----------------
+ */
+CATALOG(pg_zstd_dictionaries,9946,ZstdDictionariesRelationId)
+{
+ Oid dictid BKI_FORCE_NOT_NULL;
+
+ /*
+ * variable-length fields start here, but we allow direct access to dict
+ */
+ bytea dict BKI_FORCE_NOT_NULL;
+} FormData_pg_zstd_dictionaries;
+
+/* Pointer type to a tuple with the format of pg_zstd_dictionaries relation */
+typedef FormData_pg_zstd_dictionaries *Form_pg_zstd_dictionaries;
+
+DECLARE_TOAST(pg_zstd_dictionaries, 9947, 9948);
+
+DECLARE_UNIQUE_INDEX_PKEY(pg_zstd_dictionaries_dictid_index, 9949, ZstdDictidIndexId, pg_zstd_dictionaries, btree(dictid oid_ops));
+
+MAKE_SYSCACHE(ZSTDDICTIDOID, pg_zstd_dictionaries_dictid_index, 128);
+
+typedef struct ZstdTrainingData
+{
+ char *sample_buffer; /* Pointer to the raw sample buffer */
+ size_t *sample_sizes; /* Array of sample sizes */
+ int nitems; /* Number of sample sizes */
+} ZstdTrainingData;
+
+extern bytea *get_zstd_dict(Oid dictid);
+
+#endif /* PG_ZSTD_DICTIONARIES_H */
diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h
index f1bd18c49f..e494436870 100644
--- a/src/include/parser/analyze.h
+++ b/src/include/parser/analyze.h
@@ -17,6 +17,7 @@
#include "nodes/params.h"
#include "nodes/queryjumble.h"
#include "parser/parse_node.h"
+#include "access/htup.h"
/* Hook for plugins to get control at end of parse analysis */
typedef void (*post_parse_analyze_hook_type) (ParseState *pstate,
@@ -64,4 +65,8 @@ extern List *BuildOnConflictExcludedTargetlist(Relation targetrel,
extern SortGroupClause *makeSortGroupClauseForSetOp(Oid rescoltype, bool require_hash);
+extern int acquire_sample_rows(Relation onerel, int elevel,
+ HeapTuple *rows, int targrows,
+ double *totalrows, double *totaldeadrows);
+
#endif /* ANALYZE_H */
diff --git a/src/include/utils/attoptcache.h b/src/include/utils/attoptcache.h
index f684a772af..93c6c59e08 100644
--- a/src/include/utils/attoptcache.h
+++ b/src/include/utils/attoptcache.h
@@ -21,6 +21,12 @@ typedef struct AttributeOpts
int32 vl_len_; /* varlena header (do not touch directly!) */
float8 n_distinct;
float8 n_distinct_inherited;
+ double zstd_dictid; /* Oid is a 32-bit unsigned integer, but
+ * relopt_int is limited to INT_MAX, so it
+ * cannot represent the full range of Oid
+ * values. */
+ int zstd_dict_size;
+ int zstd_cmp_level;
} AttributeOpts;
extern AttributeOpts *get_attribute_options(Oid attrelid, int attnum);
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 2e8564d499..11e0bf53ad 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -36,14 +36,17 @@ typedef struct varatt_external
* compression method */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
+ uint32 va_cmp_alg; /* The additional compression algorithms
+ * information. */
} varatt_external;
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
*/
-#define VARLENA_EXTSIZE_BITS 30
-#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTSIZE_BITS 30
+#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTENDED_COMPRESSION_FLAG 0x3
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
@@ -122,6 +125,13 @@ typedef union
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
+ struct
+ {
+ uint32 va_header;
+ uint32 va_tcinfo;
+ uint32 va_cmp_alg;
+ char va_data[FLEXIBLE_ARRAY_MEMBER];
+ } va_compressed_ext;
} varattrib_4b;
typedef struct
@@ -242,7 +252,14 @@ typedef struct
#endif /* WORDS_BIGENDIAN */
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
-#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
+/*
+ * If va_tcinfo >> VARLENA_EXTSIZE_BITS == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * use va_compressed_ext; otherwise, use the va_compressed.
+ */
+#define VARDATA_4B_C(PTR) \
+( (((varattrib_4b *)(PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG \
+ ? ((varattrib_4b *)(PTR))->va_compressed_ext.va_data \
+ : ((varattrib_4b *)(PTR))->va_compressed.va_data )
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
@@ -252,6 +269,7 @@ typedef struct
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
+#define VARHDRSZ_COMPRESSED_EXT offsetof(varattrib_4b, va_compressed_ext.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
@@ -327,22 +345,54 @@ typedef struct
/* Decompressed size and compression method of a compressed-in-line Datum */
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
-#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
- (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
+/*
+ * - "Extended" format is indicated by (va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * - For the non-extended formats, the method code is stored in the top bits of va_tcinfo.
+ * - In the extended format, the method code is stored in va_cmp_alg instead.
+ */
+#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
+( ((((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((varattrib_4b *) (PTR))->va_compressed_ext.va_cmp_alg) \
+ : ( (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo) >> VARLENA_EXTSIZE_BITS))
/* Same for external Datums; but note argument is a struct varatt_external */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
-#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
- ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
-
-#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
- do { \
- Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
- } while (0)
+/*
+ * If ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG,
+ * that means the compression method code is in va_cmp_alg.
+ * Otherwise, we return the top two bits directly from va_extinfo.
+ */
+#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
+( ( ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (toast_pointer).va_cmp_alg \
+ : ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS) )
+
+#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
+ do { \
+ /* If desired, keep or expand the Assert checks for known methods: */ \
+ Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == TOAST_ZSTD_COMPRESSION_ID); \
+ if ((cm) < TOAST_ZSTD_COMPRESSION_ID) \
+ { \
+ /* Store the actual method in va_extinfo */ \
+ (toast_pointer).va_extinfo = (uint32)(len) \
+ | ((uint32)(cm) << VARLENA_EXTSIZE_BITS); \
+ /* Clear or set va_cmp_alg to 0 */ \
+ (toast_pointer).va_cmp_alg = 0; \
+ } \
+ else \
+ { \
+ /* Store VARLENA_EXTENDED_COMPRESSION_FLAG in the top bits, \
+ meaning "extended" method. */ \
+ (toast_pointer).va_extinfo = (uint32)(len) | \
+ ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG \
+ << VARLENA_EXTSIZE_BITS); \
+ /* Put the real method code in va_cmp_alg */ \
+ (toast_pointer).va_cmp_alg = (cm); \
+ } \
+ } while (0)
/*
* Testing whether an externally-stored value is compressed now requires
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 4dd9ee7200..94495388ad 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -238,10 +238,11 @@ NOTICE: merging multiple inherited definitions of column "f1"
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
-- test alter compression method
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 7bd7642b4b..0ce4915217 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -233,6 +233,9 @@ HINT: Available values: pglz.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
HINT: Available values: pglz.
+SET default_toast_compression = 'zstd';
+ERROR: invalid value for parameter "default_toast_compression": "zstd"
+HINT: Available values: pglz.
SET default_toast_compression = 'lz4';
ERROR: invalid value for parameter "default_toast_compression": "lz4"
HINT: Available values: pglz.
diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out
new file mode 100644
index 0000000000..7de110a90a
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd.out
@@ -0,0 +1,123 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+ERROR: compression method zstd not supported
+DETAIL: This functionality requires the server to be built with zstd support.
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 10...
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1)
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 3: FROM cmdata_zstd
+ ^
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPR...
+ ^
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+ERROR: table "cmdata_zstd_2" does not exist
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: SELECT f1 FROM cmdata_zstd;
+ ^
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ERROR: relation "compressmv_zstd" does not exist
+LINE 2: FROM compressmv_zstd;
+ ^
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: UPDATE cmdata_zstd
+ ^
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+ERROR: materialized view "compressmv_zstd" does not exist
+DROP TABLE cmdata_zstd;
+ERROR: table "cmdata_zstd" does not exist
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out
new file mode 100644
index 0000000000..a540c99b37
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd_1.out
@@ -0,0 +1,181 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+ Table "public.cmdata_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ compression_method | row_count
+--------------------+-----------
+ zstd | 20
+(1 row)
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ data_slice
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+(20 rows)
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+ Table "public.cmdata_zstd_2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+DROP TABLE cmdata_zstd_2;
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+ Materialized view "public.compressmv_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | | |
+View definition:
+ SELECT f1
+ FROM cmdata_zstd;
+
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ mv_compression
+----------------
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+(20 rows)
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ preview
+---------
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+(20 rows)
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out
index 215eb899be..ac5da3f5ab 100644
--- a/src/test/regress/expected/oidjoins.out
+++ b/src/test/regress/expected/oidjoins.out
@@ -71,6 +71,7 @@ NOTICE: checking pg_type {typmodout} => pg_proc {oid}
NOTICE: checking pg_type {typanalyze} => pg_proc {oid}
NOTICE: checking pg_type {typbasetype} => pg_type {oid}
NOTICE: checking pg_type {typcollation} => pg_collation {oid}
+NOTICE: checking pg_type {typebuildzstddictionary} => pg_proc {oid}
NOTICE: checking pg_attribute {attrelid} => pg_class {oid}
NOTICE: checking pg_attribute {atttypid} => pg_type {oid}
NOTICE: checking pg_attribute {attcollation} => pg_collation {oid}
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 37b6d21e1f..407a0644f8 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -119,7 +119,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# The stats test resets stats, so nothing else needing stats access can be in
# this group.
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats predicate
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd memoize stats predicate
# event_trigger depends on create_am and cannot run concurrently with
# any test that runs DDL
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index 490595fcfb..e29909558f 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -102,6 +102,7 @@ CREATE TABLE cminh() INHERITS (cmdata, cmdata3);
-- test default_toast_compression GUC
SET default_toast_compression = '';
SET default_toast_compression = 'I do not exist compression';
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql
new file mode 100644
index 0000000000..7cf93e3de2
--- /dev/null
+++ b/src/test/regress/sql/compression_zstd.sql
@@ -0,0 +1,97 @@
+\set HIDE_TOAST_COMPRESSION false
+
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9840060997..adb94ee7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -886,6 +886,7 @@ FormData_pg_ts_parser
FormData_pg_ts_template
FormData_pg_type
FormData_pg_user_mapping
+FormData_pg_zstd_dictionaries
FormExtraData_pg_attribute
Form_pg_aggregate
Form_pg_am
@@ -945,6 +946,7 @@ Form_pg_ts_parser
Form_pg_ts_template
Form_pg_type
Form_pg_user_mapping
+Form_pg_zstd_dictionaries
FormatNode
FreeBlockNumberArray
FreeListData
@@ -2582,6 +2584,8 @@ STARTUPINFO
STRLEN
SV
SYNCHRONIZATION_BARRIER
+SampleCollector
+SampleEntry
SampleScan
SampleScanGetSampleSize_function
SampleScanState
@@ -3306,6 +3310,7 @@ ZSTD_cParameter
ZSTD_inBuffer
ZSTD_outBuffer
ZstdCompressorState
+ZstdTrainingData
_SPI_connection
_SPI_plan
__m128i
base-commit: 39de4f157d3ac0b889bb276c2487fe160578f967
--
2.47.1
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 12:02 Kirill Reshke <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
2 siblings, 0 replies; 21+ messages in thread
From: Kirill Reshke @ 2025-03-06 12:02 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: pgsql-hackers
On Thu, 6 Mar 2025 at 08:43, Nikhil Kumar Veldanda
<[email protected]> wrote:
>
> Hi all,
>
> The ZStandard compression algorithm [1][2], though not currently used for TOAST compression in PostgreSQL, offers significantly improved compression ratios compared to lz4/pglz in both dictionary-based and non-dictionary modes. Attached find for review my patch to add ZStandard compression to Postgres. In tests this patch used with a pre-trained dictionary achieved up to four times the compression ratio of LZ4, while ZStandard without a dictionary outperformed LZ4/pglz by about two times during compression of data.
>
> Notably, this is the first compression algorithm for Postgres that can make use of a dictionary to provide higher levels of compression, but dictionaries have to be generated and maintained, and so I’ve had to break new ground in that regard. To use the dictionary support requires training and storing a dictionary for a given variable-length column type. On a variable-length column, a SQL function will be called. It will sample the column’s data and feed it into the ZStandard training API which will return a dictionary. In the example, the column is of JSONB type. The SQL function takes the table name and the attribute number as inputs. If the training is successful, it will return true; otherwise, it will return false.
>
> ‘’‘
> test=# select build_zstd_dict_for_attribute('"public"."zstd"', 1);
> build_zstd_dict_for_attribute
> -------------------------------
> t
> (1 row)
> ‘’‘
>
> The sampling logic and data to feed to the ZStandard training API can vary by data type. The patch includes an method to write other type-specific training functions and includes a default for JSONB, TEXT and BYTEA. There is a new option called ‘build_zstd_dict’ that takes a function name as input in ‘CREATE TYPE’. In this way anyone can write their own type-specific training function by handling sampling logic and returning the necessary information for the ZStandard training API in “ZstdTrainingData” format.
>
> ```
> typedef struct ZstdTrainingData
> {
> char *sample_buffer; /* Pointer to the raw sample buffer */
> size_t *sample_sizes; /* Array of sample sizes */
> int nitems; /* Number of sample sizes */
> } ZstdTrainingData;
> ```
> This information is feed into the ZStandard train API, which generates a dictionary and inserts it into the dictionary catalog table. Additionally, we update the ‘pg_attribute’ attribute options to include the unique dictionary ID for that specific attribute. During compression, based on the available dictionary ID, we retrieve the dictionary and use it to compress the documents. I’ve created standard training function (`zstd_dictionary_builder`) for JSONB, TEXT, and BYTEA.
>
> We store dictionary and dictid in the new catalog table ‘pg_zstd_dictionaries’
>
> ```
> test=# \d pg_zstd_dictionaries
> Table "pg_catalog.pg_zstd_dictionaries"
> Column | Type | Collation | Nullable | Default
> --------+-------+-----------+----------+---------
> dictid | oid | | not null |
> dict | bytea | | not null |
> Indexes:
> "pg_zstd_dictionaries_dictid_index" PRIMARY KEY, btree (dictid)
> ```
>
> This is the entire ZStandard dictionary infrastructure. A column can have multiple dictionaries. The latest dictionary will be identified by the pg_attribute attoptions. We never delete dictionaries once they are generated. If a dictionary is not provided and attcompression is set to zstd, we compress with ZStandard without dictionary. For decompression, the zstd-compressed frame contains a dictionary identifier (dictid) that indicates the dictionary used for compression. By retrieving this dictid from the zstd frame, we then fetch the corresponding dictionary and perform decompression.
>
> #############################################################################
>
> Enter toast compression framework changes,
>
> We identify a compressed datum compression algorithm using the top two bits of va_tcinfo (varattrib_4b.va_compressed).
> It is possible to have four compression methods. However, based on previous community email discussions regarding toast compression changes[3], the idea of using it for a new compression algorithm has been rejected, and a suggestion has been made to extend it which I’ve implemented in this patch. This change necessitates an update to ‘varattrib_4b’ and ‘varatt_external’ on disk structures. I’ve made sure that this changes are backward compatible.
>
> ```
> typedef union
> {
> struct /* Normal varlena (4-byte length) */
> {
> uint32 va_header;
> char va_data[FLEXIBLE_ARRAY_MEMBER];
> } va_4byte;
> struct /* Compressed-in-line format */
> {
> uint32 va_header;
> uint32 va_tcinfo; /* Original data size (excludes header) and
> * compression method; see va_extinfo */
> char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
> } va_compressed;
> struct
> {
> uint32 va_header;
> uint32 va_tcinfo;
> uint32 va_cmp_alg;
> char va_data[FLEXIBLE_ARRAY_MEMBER];
> } va_compressed_ext;
> } varattrib_4b;
>
> typedef struct varatt_external
> {
> int32 va_rawsize; /* Original data size (includes header) */
> uint32 va_extinfo; /* External saved size (without header) and
> * compression method */
> Oid va_valueid; /* Unique ID of value within TOAST table */
> Oid va_toastrelid; /* RelID of TOAST table containing it */
> uint32 va_cmp_alg; /* The additional compression algorithms
> * information. */
> } varatt_external;
> ```
>
> As I need to update this structs, I’ve made changes to the existing macros. Additionally added compression and decompression routines related to ZStandard as needed. These are major design changes in the patch to incorporate ZStandard with dictionary compression.
>
> Please let me know what you think about all this. Are there any concerns with my approach? In particular, I would appreciate your thoughts on the on-disk changes that result from this.
>
> kind regards,
>
> Nikhil Veldanda
> Amazon Web Services: https://aws.amazon.com
>
> [1] https://facebook.github.io/zstd/
> [2] https://github.com/facebook/zstd
> [3] https://www.postgresql.org/message-id/flat/YoMiNmkztrslDbNS%40paquier.xyz
>
Hi!
I generally love this idea, however I am not convinced in-core support
this is the right direction here. Maybe we can introduce some API
infrastructure here to allow delegating compression to extension's?
This is merely my opinion; perhaps dealing with a redo is not
worthwhile.
I did a brief lookup on patch v1. I feel like this is too much for a
single patch. Take, for example this change:
```
-#define NO_LZ4_SUPPORT() \
+#define NO_METHOD_SUPPORT(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with
lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with
%s support.", method)))
```
This could be a separate preliminary refactoring patch in series.
Perhaps we need to divide the patch into smaller pieces if we follow
the suggested course of this thread (in-core support).
I will try to give another in-depth look here soon.
--
Best regards,
Kirill Reshke
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 13:35 Aleksander Alekseev <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
2 siblings, 1 reply; 21+ messages in thread
From: Aleksander Alekseev @ 2025-03-06 13:35 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Nikhil Kumar Veldanda <[email protected]>
Hi Nikhil,
Many thanks for working on this. I proposed a similar patch some time
ago [1] but the overall feedback was somewhat mixed so I choose to
focus on something else. Thanks for peeking this up.
> test=# select build_zstd_dict_for_attribute('"public"."zstd"', 1);
> build_zstd_dict_for_attribute
> -------------------------------
> t
> (1 row)
Did you have a chance to familiarize yourself with the corresponding
discussion [1] and probably the previous threads? Particularly it was
pointed out that dictionaries should be built automatically during
VACUUM. We also discussed a special syntax for the feature, besides
other things.
[1]: https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22%3D5xVBg7S4vr5rQ%40m...
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 16:47 Nikhil Kumar Veldanda <[email protected]>
parent: Aleksander Alekseev <[email protected]>
0 siblings, 0 replies; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-03-06 16:47 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: pgsql-hackers
Hi
On Thu, Mar 6, 2025 at 5:35 AM Aleksander Alekseev
<[email protected]> wrote:
>
> Hi Nikhil,
>
> Many thanks for working on this. I proposed a similar patch some time
> ago [1] but the overall feedback was somewhat mixed so I choose to
> focus on something else. Thanks for peeking this up.
>
> > test=# select build_zstd_dict_for_attribute('"public"."zstd"', 1);
> > build_zstd_dict_for_attribute
> > -------------------------------
> > t
> > (1 row)
>
> Did you have a chance to familiarize yourself with the corresponding
> discussion [1] and probably the previous threads? Particularly it was
> pointed out that dictionaries should be built automatically during
> VACUUM. We also discussed a special syntax for the feature, besides
> other things.
>
> [1]: https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22%3D5xVBg7S4vr5rQ%40m...
Restricting dictionary generation to the vacuum process is not ideal
because it limits user control and flexibility. Compression efficiency
is highly dependent on data distribution, which can change
dynamically. By allowing users to generate dictionaries on demand via
an API, they can optimize compression when they detect inefficiencies
rather than waiting for a vacuum process, which may not align with
their needs.
Additionally, since all dictionaries are stored in the catalog table
anyway, users can generate and manage them independently without
interfering with the system’s automatic maintenance tasks. This
approach ensures better adaptability to real-world scenarios where
compression performance needs to be monitored and adjusted in real
time.
---
Nikhil Veldanda
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 19:15 Robert Haas <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
2 siblings, 2 replies; 21+ messages in thread
From: Robert Haas @ 2025-03-06 19:15 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: pgsql-hackers
On Thu, Mar 6, 2025 at 12:43 AM Nikhil Kumar Veldanda
<[email protected]> wrote:
> Notably, this is the first compression algorithm for Postgres that can make use of a dictionary to provide higher levels of compression, but dictionaries have to be generated and maintained,
I think that solving the problems around using a dictionary is going
to be really hard. Can we see some evidence that the results will be
worth it?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 19:33 Tom Lane <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 1 reply; 21+ messages in thread
From: Tom Lane @ 2025-03-06 19:33 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nikhil Kumar Veldanda <[email protected]>; pgsql-hackers
Robert Haas <[email protected]> writes:
> On Thu, Mar 6, 2025 at 12:43 AM Nikhil Kumar Veldanda
> <[email protected]> wrote:
>> Notably, this is the first compression algorithm for Postgres that can make use of a dictionary to provide higher levels of compression, but dictionaries have to be generated and maintained,
> I think that solving the problems around using a dictionary is going
> to be really hard. Can we see some evidence that the results will be
> worth it?
BTW, this is hardly the first such attempt. See [1] for a prior
attempt at something fairly similar, which ended up going nowhere.
It'd be wise to understand why that failed before pressing forward.
Note that the thread title for [1] is pretty misleading, as the
original discussion about JSONB-specific compression soon migrated
to discussion of compressing TOAST data using dictionaries. At
least from a ten-thousand-foot viewpoint, that seems like exactly
what you're proposing here. I see that you dismissed [1] as
irrelevant upthread, but I think you'd better look closer.
regards, tom lane
[1] https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22%3D5xVBg7S4vr5rQ%40m...
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-06 22:36 Nikhil Kumar Veldanda <[email protected]>
parent: Tom Lane <[email protected]>
0 siblings, 1 reply; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-03-06 22:36 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers
Hi Tom,
On Thu, Mar 6, 2025 at 11:33 AM Tom Lane <[email protected]> wrote:
>
> Robert Haas <[email protected]> writes:
> > On Thu, Mar 6, 2025 at 12:43 AM Nikhil Kumar Veldanda
> > <[email protected]> wrote:
> >> Notably, this is the first compression algorithm for Postgres that can make use of a dictionary to provide higher levels of compression, but dictionaries have to be generated and maintained,
>
> > I think that solving the problems around using a dictionary is going
> > to be really hard. Can we see some evidence that the results will be
> > worth it?
>
> BTW, this is hardly the first such attempt. See [1] for a prior
> attempt at something fairly similar, which ended up going nowhere.
> It'd be wise to understand why that failed before pressing forward.
>
> Note that the thread title for [1] is pretty misleading, as the
> original discussion about JSONB-specific compression soon migrated
> to discussion of compressing TOAST data using dictionaries. At
> least from a ten-thousand-foot viewpoint, that seems like exactly
> what you're proposing here. I see that you dismissed [1] as
> irrelevant upthread, but I think you'd better look closer.
>
> regards, tom lane
>
> [1] https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22%3D5xVBg7S4vr5rQ%40m...
Thank you for highlighting the previous discussion—I reviewed [1]
closely. While both methods involve dictionary-based compression, the
approach I'm proposing differs significantly.
The previous method explicitly extracted string values from JSONB and
assigned unique OIDs to each entry, resulting in distinct dictionary
entries for every unique value. In contrast, this approach directly
leverages Zstandard's dictionary training API. We provide raw data
samples to Zstd, which generates a dictionary of a specified size.
This dictionary is then stored in a catalog table and used to compress
subsequent inserts for the specific attribute it was trained on.
Key differences include:
1. No new data types are required.
2. Attributes can optionally have multiple dictionaries; the latest
dictionary is used during compression, and the exact dictionary used
during compression is retrieved and applied for decompression.
3. Compression utilizes Zstandard's trained dictionaries when available.
Additionally, I have provided an option for users to define custom
sampling and training logic, as directly passing raw buffers to the
training API may not always yield optimal results, especially for
certain custom variable-length data types. This flexibility motivates
the necessary adjustments to `pg_type`.
I would greatly appreciate your feedback or any additional suggestions
you might have.
[1] https://www.postgresql.org/message-id/flat/CAJ7c6TOtAB0z1UrksvGTStNE-herK-43bj22%3D5xVBg7S4vr5rQ%40m...
Best regards,
Nikhil Veldanda
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-07 11:42 Aleksander Alekseev <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
0 siblings, 1 reply; 21+ messages in thread
From: Aleksander Alekseev @ 2025-03-07 11:42 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Nikhil Kumar Veldanda <[email protected]>; Tom Lane <[email protected]>; Robert Haas <[email protected]>
Hi Nikhil,
> Thank you for highlighting the previous discussion—I reviewed [1]
> closely. While both methods involve dictionary-based compression, the
> approach I'm proposing differs significantly.
>
> The previous method explicitly extracted string values from JSONB and
> assigned unique OIDs to each entry, resulting in distinct dictionary
> entries for every unique value. In contrast, this approach directly
> leverages Zstandard's dictionary training API. We provide raw data
> samples to Zstd, which generates a dictionary of a specified size.
> This dictionary is then stored in a catalog table and used to compress
> subsequent inserts for the specific attribute it was trained on.
>
> [...]
You didn't read closely enough I'm afraid. As Tom pointed out, the
title of the thread is misleading. On top of that there are several
separate threads. I did my best to cross-reference them, but
apparently didn't do good enough.
Initially I proposed to add ZSON extension [1][2] to the PostgreSQL
core. However the idea evolved into TOAST improvements that don't
require a user to use special types. You may also find interesting the
related "Pluggable TOASTer" discussion [3]. The idea there was rather
different but the discussion about extending TOAST pointers so that in
the future we can use something else than ZSTD is relevant.
You will find the recent summary of the reached agreements somewhere
around this message [4], take a look at the thread a bit above and
below it.
I believe this effort is important. You can't, however, simply discard
everything that was discussed in this area for the past several years.
If you want to succeed of course. No one will look at your patch if it
doesn't account for all the previous discussions. I'm sorry, I know
it's disappointing. This being said you should have done better
research before submitting the code. You could just ask if anyone was
working on something like this before and save a lot of time.
Personally I would suggest starting with one little step toward
compression dictionaries. Particularly focusing on extendability of
TOAST pointers. You are going to need to store dictionary ids there
and allow using other compression algorithms in the future. This will
require something like a varint/utf8-like bitmask for this. See the
previous discussions.
[1]: https://github.com/afiskon/zson
[2]: https://postgr.es/m/CAJ7c6TP3fCC9TNKJBQAcEf4c%3DL7XQZ7QvuUayLgjhNQMD_5M_A%40mail.gmail.com
[3]: https://postgr.es/m/224711f9-83b7-a307-b17f-4457ab73aa0a%40sigaev.ru
[4]: https://postgr.es/m/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40mail.gmail.com
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-07 11:56 Aleksander Alekseev <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 0 replies; 21+ messages in thread
From: Aleksander Alekseev @ 2025-03-07 11:56 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Robert Haas <[email protected]>
Hi Robert,
> I think that solving the problems around using a dictionary is going
> to be really hard. Can we see some evidence that the results will be
> worth it?
Compression dictionaries give a good compression ratio (~50%) and also
increase TPS a bit (5-10%) due to better buffer cache utilization. At
least according to synthetic and not trustworthy benchmarks I did some
years ago [1]. The result may be very dependent on the actual data of
course, not to mention particular implementation of the idea.
[1]: https://github.com/afiskon/zson/blob/master/docs/benchmark.md
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-08 01:35 Nikhil Kumar Veldanda <[email protected]>
parent: Aleksander Alekseev <[email protected]>
0 siblings, 2 replies; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-03-08 01:35 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: pgsql-hackers; Tom Lane <[email protected]>; Robert Haas <[email protected]>
Hi,
I reviewed the discussions, and while most agreements focused on
changes to the toast pointer, the design I propose requires no
modifications to it. I’ve carefully considered the design choices made
previously, and I recognize Zstd’s clear advantages in compression
efficiency and performance over algorithms like PGLZ and LZ4, we can
integrate it without altering the existing toast pointer
(varatt_external) structure.
By simply using the top two bits of the va_extinfo field (setting them
to '11') in `varatt_external`, we can signal an alternative
compression algorithm, clearly distinguishing new methods from legacy
ones. The specific algorithm used would then be recorded in the
va_cmp_alg field.
This approach addresses the issues raised in the summarized thread[1]
and to leverage dictionaries for the data that can stay in-line. While
my initial patch includes modifications to toast_pointer due to a
single dependency on (pg_column_compression), those changes aren’t
strictly necessary; resolving that dependency separately would make
the overall design even less intrusive.
Here’s an illustrative structure:
```
typedef union
{
struct /* Normal varlena (4-byte length) */
{
uint32 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_4byte;
struct /* Current Compressed format */
{
uint32 va_header;
uint32 va_tcinfo; /* Original size and compression method */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
struct /* Extended compression format */
{
uint32 va_header;
uint32 va_tcinfo;
uint32 va_cmp_alg;
uint32 va_cmp_dictid;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_compressed_ext;
} varattrib_4b;
typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
uint32 va_extinfo; /* External saved size (without header) and
* compression method */ `11` indicates new compression methods.
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
} varatt_external;
```
Decompression flow remains straightforward: once a datum is identified
as external, we detoast it, then we identify the compression algorithm
using `
TOAST_COMPRESS_METHOD` macro which refers to a varattrib_4b structure
not a toast pointer. We retrieve the compression algorithm from either
va_tcinfo or va_cmp_alg based on adjusted macros, and decompress
accordingly.
In summary, integrating Zstandard into the TOAST framework in this
minimally invasive way should yield substantial benefits.
[1] https://www.postgresql.org/message-id/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40ma...
Best regards,
Nikhil Veldanda
On Fri, Mar 7, 2025 at 3:42 AM Aleksander Alekseev
<[email protected]> wrote:
>
> Hi Nikhil,
>
> > Thank you for highlighting the previous discussion—I reviewed [1]
> > closely. While both methods involve dictionary-based compression, the
> > approach I'm proposing differs significantly.
> >
> > The previous method explicitly extracted string values from JSONB and
> > assigned unique OIDs to each entry, resulting in distinct dictionary
> > entries for every unique value. In contrast, this approach directly
> > leverages Zstandard's dictionary training API. We provide raw data
> > samples to Zstd, which generates a dictionary of a specified size.
> > This dictionary is then stored in a catalog table and used to compress
> > subsequent inserts for the specific attribute it was trained on.
> >
> > [...]
>
> You didn't read closely enough I'm afraid. As Tom pointed out, the
> title of the thread is misleading. On top of that there are several
> separate threads. I did my best to cross-reference them, but
> apparently didn't do good enough.
>
> Initially I proposed to add ZSON extension [1][2] to the PostgreSQL
> core. However the idea evolved into TOAST improvements that don't
> require a user to use special types. You may also find interesting the
> related "Pluggable TOASTer" discussion [3]. The idea there was rather
> different but the discussion about extending TOAST pointers so that in
> the future we can use something else than ZSTD is relevant.
>
> You will find the recent summary of the reached agreements somewhere
> around this message [4], take a look at the thread a bit above and
> below it.
>
> I believe this effort is important. You can't, however, simply discard
> everything that was discussed in this area for the past several years.
> If you want to succeed of course. No one will look at your patch if it
> doesn't account for all the previous discussions. I'm sorry, I know
> it's disappointing. This being said you should have done better
> research before submitting the code. You could just ask if anyone was
> working on something like this before and save a lot of time.
>
> Personally I would suggest starting with one little step toward
> compression dictionaries. Particularly focusing on extendability of
> TOAST pointers. You are going to need to store dictionary ids there
> and allow using other compression algorithms in the future. This will
> require something like a varint/utf8-like bitmask for this. See the
> previous discussions.
>
> [1]: https://github.com/afiskon/zson
> [2]: https://postgr.es/m/CAJ7c6TP3fCC9TNKJBQAcEf4c%3DL7XQZ7QvuUayLgjhNQMD_5M_A%40mail.gmail.com
> [3]: https://postgr.es/m/224711f9-83b7-a307-b17f-4457ab73aa0a%40sigaev.ru
> [4]: https://postgr.es/m/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40mail.gmail.com
>
> --
> Best regards,
> Aleksander Alekseev
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-08 16:28 Nikhil Kumar Veldanda <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
1 sibling, 0 replies; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-03-08 16:28 UTC (permalink / raw)
To: Robert Haas <[email protected]>; Tom Lane <[email protected]>; Aleksander Alekseev <[email protected]>; +Cc: pgsql-hackers
Hi all,
Attached an updated version of the patch. Specifically, I've removed
changes related to the TOAST pointer structure. This proposal is
different from earlier discussions on this topic[1], where extending
the TOAST pointer was considered essential for enabling
dictionary-based compression.
Key improvements introduced in this proposal:
1. No Changes to TOAST Pointer: The existing TOAST pointer structure
remains untouched, simplifying integration and minimizing potential
disruptions.
2. Extensible Design: The solution is structured to seamlessly
incorporate future compression algorithms beyond zstd [2], providing
greater flexibility and future-proofing.
3. Inline Data Compression with Dictionary Support: Crucially, this
approach supports dictionary-based compression for inline data.
Dictionaries are highly effective for compressing small-sized
documents, providing substantial storage savings. Please refer to the
attached image from the zstd README[2] for supporting evidence.
Omitting dictionary-based compression for inline data would
significantly reduce these benefits. For example, under previous
design constraints [3], if a 16KB document compressed down to 256
bytes using a dictionary, storing this inline would not have been
feasible. The current proposal addresses this limitation, thereby
fully leveraging dictionary-based compression.
I believe this solution effectively addresses the limitations
identified in our earlier discussions [1][3].
Feedback on this approach would be greatly appreciated, I welcome any
feedback or suggestions you might have.
References:
[1] https://www.postgresql.org/message-id/flat/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w...
[2] https://github.com/facebook/zstd
[3] https://www.postgresql.org/message-id/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40ma...
```
typedef union
{
struct /* Normal varlena (4-byte length) */
{
uint32 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_4byte;
struct /* Compressed-in-line format */
{
uint32 va_header;
uint32 va_tcinfo; /* Original data size (excludes header) and
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
struct
{
uint32 va_header;
uint32 va_tcinfo;
uint32 va_cmp_alg;
uint32 va_cmp_dictid;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_compressed_ext;
} varattrib_4b;
```
Additional algorithm information and dictid is stored in varattrib_4b.
Best regards,
Nikhil Veldanda
On Fri, Mar 7, 2025 at 5:35 PM Nikhil Kumar Veldanda
<[email protected]> wrote:
>
> Hi,
>
> I reviewed the discussions, and while most agreements focused on
> changes to the toast pointer, the design I propose requires no
> modifications to it. I’ve carefully considered the design choices made
> previously, and I recognize Zstd’s clear advantages in compression
> efficiency and performance over algorithms like PGLZ and LZ4, we can
> integrate it without altering the existing toast pointer
> (varatt_external) structure.
>
> By simply using the top two bits of the va_extinfo field (setting them
> to '11') in `varatt_external`, we can signal an alternative
> compression algorithm, clearly distinguishing new methods from legacy
> ones. The specific algorithm used would then be recorded in the
> va_cmp_alg field.
>
> This approach addresses the issues raised in the summarized thread[1]
> and to leverage dictionaries for the data that can stay in-line. While
> my initial patch includes modifications to toast_pointer due to a
> single dependency on (pg_column_compression), those changes aren’t
> strictly necessary; resolving that dependency separately would make
> the overall design even less intrusive.
>
> Here’s an illustrative structure:
> ```
> typedef union
> {
> struct /* Normal varlena (4-byte length) */
> {
> uint32 va_header;
> char va_data[FLEXIBLE_ARRAY_MEMBER];
> } va_4byte;
> struct /* Current Compressed format */
> {
> uint32 va_header;
> uint32 va_tcinfo; /* Original size and compression method */
> char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
> } va_compressed;
> struct /* Extended compression format */
> {
> uint32 va_header;
> uint32 va_tcinfo;
> uint32 va_cmp_alg;
> uint32 va_cmp_dictid;
> char va_data[FLEXIBLE_ARRAY_MEMBER];
> } va_compressed_ext;
> } varattrib_4b;
>
> typedef struct varatt_external
> {
> int32 va_rawsize; /* Original data size (includes header) */
> uint32 va_extinfo; /* External saved size (without header) and
> * compression method */ `11` indicates new compression methods.
> Oid va_valueid; /* Unique ID of value within TOAST table */
> Oid va_toastrelid; /* RelID of TOAST table containing it */
> } varatt_external;
> ```
>
> Decompression flow remains straightforward: once a datum is identified
> as external, we detoast it, then we identify the compression algorithm
> using `
> TOAST_COMPRESS_METHOD` macro which refers to a varattrib_4b structure
> not a toast pointer. We retrieve the compression algorithm from either
> va_tcinfo or va_cmp_alg based on adjusted macros, and decompress
> accordingly.
>
> In summary, integrating Zstandard into the TOAST framework in this
> minimally invasive way should yield substantial benefits.
>
> [1] https://www.postgresql.org/message-id/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40ma...
>
> Best regards,
> Nikhil Veldanda
>
> On Fri, Mar 7, 2025 at 3:42 AM Aleksander Alekseev
> <[email protected]> wrote:
> >
> > Hi Nikhil,
> >
> > > Thank you for highlighting the previous discussion—I reviewed [1]
> > > closely. While both methods involve dictionary-based compression, the
> > > approach I'm proposing differs significantly.
> > >
> > > The previous method explicitly extracted string values from JSONB and
> > > assigned unique OIDs to each entry, resulting in distinct dictionary
> > > entries for every unique value. In contrast, this approach directly
> > > leverages Zstandard's dictionary training API. We provide raw data
> > > samples to Zstd, which generates a dictionary of a specified size.
> > > This dictionary is then stored in a catalog table and used to compress
> > > subsequent inserts for the specific attribute it was trained on.
> > >
> > > [...]
> >
> > You didn't read closely enough I'm afraid. As Tom pointed out, the
> > title of the thread is misleading. On top of that there are several
> > separate threads. I did my best to cross-reference them, but
> > apparently didn't do good enough.
> >
> > Initially I proposed to add ZSON extension [1][2] to the PostgreSQL
> > core. However the idea evolved into TOAST improvements that don't
> > require a user to use special types. You may also find interesting the
> > related "Pluggable TOASTer" discussion [3]. The idea there was rather
> > different but the discussion about extending TOAST pointers so that in
> > the future we can use something else than ZSTD is relevant.
> >
> > You will find the recent summary of the reached agreements somewhere
> > around this message [4], take a look at the thread a bit above and
> > below it.
> >
> > I believe this effort is important. You can't, however, simply discard
> > everything that was discussed in this area for the past several years.
> > If you want to succeed of course. No one will look at your patch if it
> > doesn't account for all the previous discussions. I'm sorry, I know
> > it's disappointing. This being said you should have done better
> > research before submitting the code. You could just ask if anyone was
> > working on something like this before and save a lot of time.
> >
> > Personally I would suggest starting with one little step toward
> > compression dictionaries. Particularly focusing on extendability of
> > TOAST pointers. You are going to need to store dictionary ids there
> > and allow using other compression algorithms in the future. This will
> > require something like a varint/utf8-like bitmask for this. See the
> > previous discussions.
> >
> > [1]: https://github.com/afiskon/zson
> > [2]: https://postgr.es/m/CAJ7c6TP3fCC9TNKJBQAcEf4c%3DL7XQZ7QvuUayLgjhNQMD_5M_A%40mail.gmail.com
> > [3]: https://postgr.es/m/224711f9-83b7-a307-b17f-4457ab73aa0a%40sigaev.ru
> > [4]: https://postgr.es/m/CAJ7c6TPSN06C%2B5cYSkyLkQbwN1C%2BpUNGmx%2BVoGCA-SPLCszC8w%40mail.gmail.com
> >
> > --
> > Best regards,
> > Aleksander Alekseev
Attachments:
[application/octet-stream] v2-0001-Add-ZStandard-with-dictionaries-compression-suppo.patch (103.6K, ../../CAFAfj_FbdeabZEqU_vCCLar07DvF0SJRgimW8A7ZmD=UPwE6VA@mail.gmail.com/2-v2-0001-Add-ZStandard-with-dictionaries-compression-suppo.patch)
download | inline diff:
From bb4d6120cc466a515268ddc80e7229f1b00f0801 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Sat, 8 Mar 2025 13:44:38 +0000
Subject: [PATCH v2] Add ZStandard (with dictionaries) compression support for
TOAST
---
contrib/amcheck/verify_heapam.c | 1 +
doc/src/sgml/catalogs.sgml | 55 ++
doc/src/sgml/ref/create_type.sgml | 21 +-
src/backend/access/brin/brin_tuple.c | 21 +-
src/backend/access/common/detoast.c | 12 +-
src/backend/access/common/indextuple.c | 16 +-
src/backend/access/common/reloptions.c | 36 +-
src/backend/access/common/toast_compression.c | 275 ++++++++-
src/backend/access/common/toast_internals.c | 17 +-
src/backend/access/table/toast_helper.c | 21 +-
src/backend/catalog/Makefile | 3 +-
src/backend/catalog/heap.c | 8 +-
src/backend/catalog/meson.build | 1 +
src/backend/catalog/pg_type.c | 11 +-
src/backend/catalog/pg_zstd_dictionaries.c | 566 ++++++++++++++++++
src/backend/commands/analyze.c | 7 +-
src/backend/commands/typecmds.c | 99 ++-
src/backend/utils/adt/varlena.c | 3 +
src/backend/utils/misc/guc_tables.c | 3 +
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/bin/pg_dump/pg_dump.c | 25 +-
src/bin/psql/describe.c | 5 +-
src/include/access/toast_compression.h | 13 +-
src/include/access/toast_helper.h | 2 +
src/include/access/toast_internals.h | 51 +-
src/include/catalog/Makefile | 3 +-
src/include/catalog/catversion.h | 2 +-
src/include/catalog/meson.build | 1 +
src/include/catalog/pg_proc.dat | 10 +
src/include/catalog/pg_type.dat | 6 +-
src/include/catalog/pg_type.h | 8 +-
src/include/catalog/pg_zstd_dictionaries.h | 53 ++
src/include/parser/analyze.h | 5 +
src/include/utils/attoptcache.h | 6 +
src/include/varatt.h | 67 ++-
src/test/regress/expected/compression.out | 5 +-
src/test/regress/expected/compression_1.out | 3 +
.../regress/expected/compression_zstd.out | 123 ++++
.../regress/expected/compression_zstd_1.out | 181 ++++++
src/test/regress/expected/oidjoins.out | 1 +
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/compression.sql | 1 +
src/test/regress/sql/compression_zstd.sql | 97 +++
src/tools/pgindent/typedefs.list | 5 +
44 files changed, 1763 insertions(+), 90 deletions(-)
create mode 100644 src/backend/catalog/pg_zstd_dictionaries.c
create mode 100644 src/include/catalog/pg_zstd_dictionaries.h
create mode 100644 src/test/regress/expected/compression_zstd.out
create mode 100644 src/test/regress/expected/compression_zstd_1.out
create mode 100644 src/test/regress/sql/compression_zstd.sql
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 827312306f..f01cc940e3 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1700,6 +1700,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* List of all valid compression method IDs */
case TOAST_PGLZ_COMPRESSION_ID:
case TOAST_LZ4_COMPRESSION_ID:
+ case TOAST_ZSTD_COMPRESSION_ID:
valid = true;
break;
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index fb05063555..ed4c51a678 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -369,6 +369,12 @@
<entry><link linkend="catalog-pg-user-mapping"><structname>pg_user_mapping</structname></link></entry>
<entry>mappings of users to foreign servers</entry>
</row>
+
+ <row>
+ <entry><link linkend="catalog-pg-zstd-dictionaries"><structname>pg_zstd_dictionaries</structname></link></entry>
+ <entry>Zstandard dictionaries</entry>
+ </row>
+
</tbody>
</tgroup>
</table>
@@ -9779,4 +9785,53 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</table>
</sect1>
+
+<sect1 id="catalog-pg-zstd-dictionaries">
+ <title><structname>pg_zstd_dictionaries</structname></title>
+
+ <indexterm zone="catalog-pg-zstd-dictionaries">
+ <primary>pg_zstd_dictionaries</primary>
+ </indexterm>
+
+ <para>
+ The catalog <structname>pg_zstd_dictionaries</structname> maintains the dictionaries essential for Zstandard compression and decompression.
+ </para>
+
+ <table>
+ <title><structname>pg_zstd_dictionaries</structname> Columns</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">Column Type</para>
+ <para>Description</para>
+ </entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dictid</structfield> <type>oid</type>
+ </para>
+ <para>
+ Dictionary identifier; a non-null OID that uniquely identifies a dictionary.
+ </para>
+ </entry>
+ </row>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dict</structfield> <type>bytea</type>
+ </para>
+ <para>
+ Variable-length field containing the zstd dictionary data. This field must not be null.
+ </para>
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+</sect1>
+
</chapter>
diff --git a/doc/src/sgml/ref/create_type.sgml b/doc/src/sgml/ref/create_type.sgml
index 994dfc6526..ad4cf2f8b3 100644
--- a/doc/src/sgml/ref/create_type.sgml
+++ b/doc/src/sgml/ref/create_type.sgml
@@ -56,6 +56,7 @@ CREATE TYPE <replaceable class="parameter">name</replaceable> (
[ , ELEMENT = <replaceable class="parameter">element</replaceable> ]
[ , DELIMITER = <replaceable class="parameter">delimiter</replaceable> ]
[ , COLLATABLE = <replaceable class="parameter">collatable</replaceable> ]
+ [ , BUILD_ZSTD_DICT = <replaceable class="parameter">zstd_training_function</replaceable> ]
)
CREATE TYPE <replaceable class="parameter">name</replaceable>
@@ -211,7 +212,8 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
<replaceable class="parameter">type_modifier_input_function</replaceable>,
<replaceable class="parameter">type_modifier_output_function</replaceable>,
<replaceable class="parameter">analyze_function</replaceable>, and
- <replaceable class="parameter">subscript_function</replaceable>
+ <replaceable class="parameter">subscript_function</replaceable>, and
+ <replaceable class="parameter">zstd_training_function</replaceable>
are optional. Generally these functions have to be coded in C
or another low-level language.
</para>
@@ -491,6 +493,15 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
make use of the collation information; this does not happen
automatically merely by marking the type collatable.
</para>
+
+ <para>
+ The optional <replaceable class="parameter">zstd_training_function</replaceable>
+ performs type-specific sample collection for a column of the corresponding data type.
+ By default, for <type>jsonb</type>, <type>text</type>, and <type>bytea</type> data types, the function <literal>zstd_dictionary_builder</literal> is defined. It attempts to gather samples for a column
+ and returns a sample buffer for zstd dictionary training. The training function must be declared to accept two arguments of type <type>internal</type> and return an <type>internal</type> result.
+ The detailed information for zstd training function is provided in <filename>src/backend/catalog/pg_zstd_dictionaries.c</filename>.
+ </para>
+
</refsect2>
<refsect2 id="sql-createtype-array" xreflabel="Array Types">
@@ -846,6 +857,14 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
</para>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><replaceable class="parameter">build_zstd_dict</replaceable></term>
+ <listitem>
+ <para>
+ Specifies the name of a function that performs sampling and provides the logic necessary to generate a sample buffer for zstd training.
+ </para>
+ </listitem>
+ </varlistentry>
</variablelist>
</refsect1>
diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c
index 861f397e6d..02f1996ede 100644
--- a/src/backend/access/brin/brin_tuple.c
+++ b/src/backend/access/brin/brin_tuple.c
@@ -40,7 +40,7 @@
#include "access/tupmacs.h"
#include "utils/datum.h"
#include "utils/memutils.h"
-
+#include "utils/attoptcache.h"
/*
* This enables de-toasting of index entries. Needed until VACUUM is
@@ -222,7 +222,8 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
atttype->typstorage == TYPSTORAGE_MAIN))
{
Datum cvalue;
- char compression;
+ CompressionInfo cmp = {.cmethod = InvalidCompressionMethod,.dictid = InvalidDictId,.zstd_level = DEFAULT_ZSTD_LEVEL};
+
Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
keyno);
@@ -233,11 +234,19 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
* default method.
*/
if (att->atttypid == atttype->type_id)
- compression = att->attcompression;
- else
- compression = InvalidCompressionMethod;
+ cmp.cmethod = att->attcompression;
- cvalue = toast_compress_datum(value, compression);
+ if (cmp.cmethod == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ {
+ cmp.zstd_level = aopt->zstd_level;
+ cmp.dictid = (Oid) aopt->dictid;
+ }
+ }
+ cvalue = toast_compress_datum(value, cmp);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 6265178774..b57a9f024c 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr,
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or Zstandard data, we'll have to
+ * fetch the whole thing, because there doesn't seem to be an API
+ * call to determine how much compressed data we need to be sure
+ * of being able to decompress the required slice.
*/
if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
TOAST_PGLZ_COMPRESSION_ID)
@@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index 1986b943a2..9cf5aabf51 100644
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -21,6 +21,7 @@
#include "access/htup_details.h"
#include "access/itup.h"
#include "access/toast_internals.h"
+#include "utils/attoptcache.h"
/*
* This enables de-toasting of index entries. Needed until VACUUM is
@@ -124,8 +125,19 @@ index_form_tuple_context(TupleDesc tupleDescriptor,
{
Datum cvalue;
- cvalue = toast_compress_datum(untoasted_values[i],
- att->attcompression);
+ CompressionInfo cmp = {.cmethod = att->attcompression,.dictid = InvalidDictId,.zstd_level = DEFAULT_ZSTD_LEVEL};
+
+ if (cmp.cmethod == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ {
+ cmp.zstd_level = aopt->zstd_level;
+ cmp.dictid = (Oid) aopt->dictid;
+ }
+ }
+ cvalue = toast_compress_datum(untoasted_values[i], cmp);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 59fb53e770..7c71fb3492 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -34,6 +34,7 @@
#include "utils/guc.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "access/toast_compression.h"
/*
* Contents of pg_class.reloptions
@@ -389,7 +390,26 @@ static relopt_int intRelOpts[] =
},
-1, 0, 1024
},
-
+ {
+ {
+ "zstd_dict_size",
+ "Max dict size for zstd",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_DICT_SIZE, 0, 112640 /* Max dict size(110 KB), 0
+ * indicates Don't use dictionary
+ * for compression */
+ },
+ {
+ {
+ "zstd_level",
+ "Set column's ZSTD compression level",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_LEVEL, 1, 22
+ },
/* list terminator */
{{NULL}}
};
@@ -478,6 +498,15 @@ static relopt_real realRelOpts[] =
},
0, -1.0, DBL_MAX
},
+ {
+ {
+ "dictid",
+ "Current dictid for column",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ InvalidDictId, InvalidDictId, UINT32_MAX
+ },
{
{
"vacuum_cleanup_index_scale_factor",
@@ -2093,7 +2122,10 @@ attribute_reloptions(Datum reloptions, bool validate)
{
static const relopt_parse_elt tab[] = {
{"n_distinct", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct)},
- {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)}
+ {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)},
+ {"dictid", RELOPT_TYPE_REAL, offsetof(AttributeOpts, dictid)},
+ {"zstd_dict_size", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_dict_size)},
+ {"zstd_level", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_level)},
};
return (bytea *) build_reloptions(reloptions, validate,
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 21f2f4af97..2a271b8bcd 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -17,19 +17,26 @@
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#include <zdict.h>
+#endif
+
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"
+#include "catalog/pg_zstd_dictionaries.h"
+#include "access/toast_internals.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
-#define NO_LZ4_SUPPORT() \
+#define NO_METHOD_SUPPORT(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with %s support.", method)))
/*
* Compress a varlena using PGLZ.
@@ -139,7 +146,7 @@ struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 valsize;
@@ -182,7 +189,7 @@ struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -215,7 +222,7 @@ struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -266,7 +273,13 @@ toast_get_compression_id(struct varlena *attr)
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) >= TOAST_ZSTD_COMPRESSION_ID)
+ {
+ struct varlena *compressed_attr = detoast_external_attr(attr);
+
+ cmid = TOAST_COMPRESS_METHOD(compressed_attr);
+ }
+ else
cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
}
else if (VARATT_IS_COMPRESSED(attr))
@@ -289,10 +302,17 @@ CompressionNameToMethod(const char *compression)
else if (strcmp(compression, "lz4") == 0)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ NO_METHOD_SUPPORT("lz4");
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd") == 0)
+ {
+#ifndef USE_ZSTD
+ NO_METHOD_SUPPORT("zstd");
+#endif
+ return TOAST_ZSTD_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -309,8 +329,247 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_COMPRESSION:
+ return "zstd";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
}
}
+
+/* Compress datum using ZSTD with optional dictionary (using cdict) */
+struct varlena *
+zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level)
+{
+#ifdef USE_ZSTD
+ uint32 valsize = VARSIZE_ANY_EXHDR(value);
+ size_t max_size = ZSTD_compressBound(valsize);
+ struct varlena *compressed;
+ void *dest;
+ size_t cmp_size,
+ ret;
+ ZSTD_CCtx *cctx = ZSTD_createCCtx();
+ ZSTD_CDict *cdict = NULL;
+
+ if (!cctx)
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression context")));
+
+ ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD compression level: %s", ZSTD_getErrorName(ret))));
+ }
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ cdict = ZSTD_createCDict(dict_buffer, dict_size, zstd_level);
+ pfree(dict_bytea);
+
+ if (!cdict)
+ {
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_CCtx_refCDict(cctx, cdict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeCDict(cdict);
+ ZSTD_freeCCtx(cctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ /* Allocate space for the compressed varlena (header + data) */
+ compressed = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED_EXT);
+ dest = (char *) compressed + VARHDRSZ_COMPRESSED_EXT;
+
+ /* Compress the data */
+ cmp_size = ZSTD_compress2(cctx, dest, max_size, VARDATA_ANY(value), valsize);
+
+ /* Cleanup */
+ ZSTD_freeCDict(cdict);
+ ZSTD_freeCCtx(cctx);
+
+ if (ZSTD_isError(cmp_size))
+ {
+ pfree(compressed);
+ ereport(ERROR, (errmsg("ZSTD compression failed: %s", ZSTD_getErrorName(cmp_size))));
+ }
+
+ /*
+ * If compression did not reduce size, return NULL so that the
+ * uncompressed data is stored
+ */
+ if (cmp_size > valsize)
+ {
+ pfree(compressed);
+ return NULL;
+ }
+
+ /* Set the compressed size in the varlena header */
+ SET_VARSIZE_COMPRESSED(compressed, cmp_size + VARHDRSZ_COMPRESSED_EXT);
+ return compressed;
+
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
+
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifdef USE_ZSTD
+ uint32 actual_size_exhdr = VARDATA_COMPRESSED_GET_EXTSIZE(value);
+ uint32 cmp_size_exhdr = VARSIZE_4B(value) - VARHDRSZ_COMPRESSED_EXT;
+ Oid dictid;
+ struct varlena *result;
+ size_t uncmp_size,
+ ret;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+ ZSTD_DDict *ddict = NULL;
+
+ if (!dctx)
+ ereport(ERROR, (errmsg("Failed to create ZSTD decompression context")));
+
+
+ dictid = (Oid) VARDATA_COMPRESSED_GET_DICTID(value);
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ ddict = ZSTD_createDDict(dict_buffer, dict_size);
+ pfree(dict_bytea);
+
+ if (!ddict)
+ {
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_DCtx_refDDict(dctx, ddict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ /* Allocate space for the uncompressed data */
+ result = (struct varlena *) palloc(actual_size_exhdr + VARHDRSZ);
+
+ uncmp_size = ZSTD_decompressDCtx(dctx,
+ VARDATA(result),
+ actual_size_exhdr,
+ VARDATA_4B_C(value),
+ cmp_size_exhdr);
+
+ /* Cleanup */
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+
+ if (ZSTD_isError(uncmp_size))
+ {
+ pfree(result);
+ ereport(ERROR, (errmsg("ZSTD decompression failed: %s", ZSTD_getErrorName(uncmp_size))));
+ }
+
+ /* Set final size in the varlena header */
+ SET_VARSIZE(result, uncmp_size + VARHDRSZ);
+ return result;
+
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
+
+/* Decompress a slice of the datum using the streaming API and optional dictionary */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+#ifdef USE_ZSTD
+ struct varlena *result;
+ ZSTD_inBuffer inBuf;
+ ZSTD_outBuffer outBuf;
+ ZSTD_DCtx *dctx = ZSTD_createDCtx();
+ ZSTD_DDict *ddict = NULL;
+ Oid dictid;
+ uint32 cmp_size_exhdr = VARSIZE_4B(value) - VARHDRSZ_COMPRESSED_EXT;
+ size_t ret;
+
+ if (dctx == NULL)
+ elog(ERROR, "could not create zstd decompression context");
+
+ /* Extract the dictionary ID from the compressed frame */
+ dictid = (Oid) ZSTD_getDictID_fromFrame(VARDATA_4B_C(value), cmp_size_exhdr);
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+
+ /* Create and bind the dictionary to the decompression context */
+ ddict = ZSTD_createDDict(dict_buffer, dict_size);
+ pfree(dict_bytea);
+
+ if (!ddict)
+ {
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+ }
+
+ ret = ZSTD_DCtx_refDDict(dctx, ddict);
+ if (ZSTD_isError(ret))
+ {
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ ereport(ERROR, (errmsg("Failed to reference ZSTD dictionary: %s", ZSTD_getErrorName(ret))));
+ }
+ }
+
+ inBuf.src = (char *) value + VARHDRSZ_COMPRESSED_EXT;
+ inBuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED_EXT;
+ inBuf.pos = 0;
+
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+ outBuf.dst = (char *) result + VARHDRSZ;
+ outBuf.size = slicelength;
+ outBuf.pos = 0;
+
+ /* Common decompression loop */
+ while (inBuf.pos < inBuf.size && outBuf.pos < outBuf.size)
+ {
+ ret = ZSTD_decompressStream(dctx, &outBuf, &inBuf);
+ if (ZSTD_isError(ret))
+ {
+ pfree(result);
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+ elog(ERROR, "zstd decompression failed: %s", ZSTD_getErrorName(ret));
+ }
+ }
+
+ /* Cleanup */
+ ZSTD_freeDDict(ddict);
+ ZSTD_freeDCtx(dctx);
+
+ Assert(outBuf.size == slicelength && outBuf.pos == slicelength);
+ SET_VARSIZE(result, outBuf.pos + VARHDRSZ);
+ return result;
+#else
+ NO_METHOD_SUPPORT("zstd");
+ return NULL;
+#endif
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 7d8be8346c..e6c877fd0a 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -43,11 +43,12 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
* ----------
*/
Datum
-toast_compress_datum(Datum value, char cmethod)
+toast_compress_datum(Datum value, CompressionInfo cmp)
{
struct varlena *tmp = NULL;
int32 valsize;
ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
+ uint32 dictid = cmp.dictid;
Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
@@ -55,13 +56,13 @@ toast_compress_datum(Datum value, char cmethod)
valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
/* If the compression method is not valid, use the current default */
- if (!CompressionMethodIsValid(cmethod))
- cmethod = default_toast_compression;
+ if (!CompressionMethodIsValid(cmp.cmethod))
+ cmp.cmethod = default_toast_compression;
/*
* Call appropriate compression routine for the compression method.
*/
- switch (cmethod)
+ switch (cmp.cmethod)
{
case TOAST_PGLZ_COMPRESSION:
tmp = pglz_compress_datum((const struct varlena *) value);
@@ -71,8 +72,12 @@ toast_compress_datum(Datum value, char cmethod)
tmp = lz4_compress_datum((const struct varlena *) value);
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_COMPRESSION:
+ tmp = zstd_compress_datum((const struct varlena *) value, cmp.dictid, cmp.zstd_level);
+ cmid = TOAST_ZSTD_COMPRESSION_ID;
+ break;
default:
- elog(ERROR, "invalid compression method %c", cmethod);
+ elog(ERROR, "invalid compression method %c", cmp.cmethod);
}
if (tmp == NULL)
@@ -92,7 +97,7 @@ toast_compress_datum(Datum value, char cmethod)
{
/* successful compression */
Assert(cmid != TOAST_INVALID_COMPRESSION_ID);
- TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid);
+ TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid, dictid);
return PointerGetDatum(tmp);
}
else
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index b60fab0a4d..968dd9f7c0 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -19,7 +19,8 @@
#include "access/toast_internals.h"
#include "catalog/pg_type_d.h"
#include "varatt.h"
-
+#include "utils/attoptcache.h"
+#include "access/toast_compression.h"
/*
* Prepare to TOAST a tuple.
@@ -55,6 +56,18 @@ toast_tuple_init(ToastTupleContext *ttc)
ttc->ttc_attr[i].tai_colflags = 0;
ttc->ttc_attr[i].tai_oldexternal = NULL;
ttc->ttc_attr[i].tai_compression = att->attcompression;
+ ttc->ttc_attr[i].dictid = InvalidDictId;
+ ttc->ttc_attr[i].zstd_level = DEFAULT_ZSTD_LEVEL;
+ if (att->attcompression == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt)
+ {
+ ttc->ttc_attr[i].dictid = (Oid) aopt->dictid;
+ ttc->ttc_attr[i].zstd_level = aopt->zstd_level;
+ }
+ }
if (ttc->ttc_oldvalues != NULL)
{
@@ -230,7 +243,11 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
Datum new_value;
ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
- new_value = toast_compress_datum(*value, attr->tai_compression);
+ CompressionInfo cmp = {.cmethod = attr->tai_compression,
+ .dictid = attr->dictid,
+ .zstd_level = attr->zstd_level};
+
+ new_value = toast_compress_datum(*value, cmp);
if (DatumGetPointer(new_value) != NULL)
{
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index c090094ed0..282afbcef5 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -46,7 +46,8 @@ OBJS = \
pg_subscription.o \
pg_type.o \
storage.o \
- toasting.o
+ toasting.o \
+ pg_zstd_dictionaries.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index bd3554c0bf..493963b1b8 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -1071,7 +1071,9 @@ AddNewRelationType(const char *typeName,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ InvalidOid /* generate dictionary procedure - default */
+ );
}
/* --------------------------------
@@ -1394,7 +1396,9 @@ heap_create_with_catalog(const char *relname,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ InvalidOid /* generate dictionary procedure - default */
+ );
pfree(relarrayname);
}
diff --git a/src/backend/catalog/meson.build b/src/backend/catalog/meson.build
index 1958ea9238..8f0413189c 100644
--- a/src/backend/catalog/meson.build
+++ b/src/backend/catalog/meson.build
@@ -34,6 +34,7 @@ backend_sources += files(
'pg_type.c',
'storage.c',
'toasting.c',
+ 'pg_zstd_dictionaries.c',
)
diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c
index b36f81afb9..bbed8f64ad 100644
--- a/src/backend/catalog/pg_type.c
+++ b/src/backend/catalog/pg_type.c
@@ -120,6 +120,7 @@ TypeShellMake(const char *typeName, Oid typeNamespace, Oid ownerId)
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(-1);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(0);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(InvalidOid);
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(InvalidOid);
nulls[Anum_pg_type_typdefaultbin - 1] = true;
nulls[Anum_pg_type_typdefault - 1] = true;
nulls[Anum_pg_type_typacl - 1] = true;
@@ -223,7 +224,8 @@ TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims, /* Array dimensions for baseType */
bool typeNotNull,
- Oid typeCollation)
+ Oid typeCollation,
+ Oid generateDictionaryProcedure)
{
Relation pg_type_desc;
Oid typeObjectId;
@@ -378,6 +380,7 @@ TypeCreate(Oid newTypeOid,
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(typeMod);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(typNDims);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(typeCollation);
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(generateDictionaryProcedure);
/*
* initialize the default binary value for this type. Check for nulls of
@@ -679,6 +682,12 @@ GenerateTypeDependencies(HeapTuple typeTuple,
add_exact_object_address(&referenced, addrs_normal);
}
+ if (OidIsValid(typeForm->typebuildzstddictionary))
+ {
+ ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typebuildzstddictionary);
+ add_exact_object_address(&referenced, addrs_normal);
+ }
+
if (OidIsValid(typeForm->typsubscript))
{
ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typsubscript);
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
new file mode 100644
index 0000000000..2d27f58221
--- /dev/null
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -0,0 +1,566 @@
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "access/heapam.h"
+#include "access/table.h"
+#include "access/relation.h"
+#include "access/tableam.h"
+#include "catalog/catalog.h"
+#include "catalog/dependency.h"
+#include "catalog/indexing.h"
+#include "catalog/pg_class_d.h"
+#include "catalog/pg_zstd_dictionaries.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+#include "catalog/pg_type.h"
+#include "catalog/namespace.h"
+#include "utils/builtins.h"
+#include "utils/fmgroids.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/hsearch.h"
+#include "access/toast_compression.h"
+#include "utils/attoptcache.h"
+#include "parser/analyze.h"
+#include "common/hashfn.h"
+#include "nodes/makefuncs.h"
+#include "access/reloptions.h"
+#include "miscadmin.h"
+#include "access/genam.h"
+#include "executor/tuptable.h"
+#include "access/htup_details.h"
+#include "access/sdir.h"
+#include "utils/lsyscache.h"
+#include "utils/relcache.h"
+#include "utils/memutils.h"
+#include "utils/varlena.h"
+#include "nodes/pg_list.h"
+
+#ifdef USE_ZSTD
+#include <zstd.h>
+#include <zdict.h>
+#endif
+
+#define TARG_ROWS 1000
+
+typedef struct SampleEntry SampleEntry;
+typedef struct SampleCollector SampleCollector;
+
+/* Structure to store a sample entry */
+struct SampleEntry
+{
+ void *data; /* Pointer to sample data */
+ size_t size; /* Size of the sample */
+};
+
+/* Structure to collect samples along with a hash table for deduplication */
+struct SampleCollector
+{
+ SampleEntry *samples; /* Dynamic array of pointers to SampleEntry */
+ int sample_count; /* Number of collected samples */
+};
+
+static bool build_zstd_dictionary_internal(Oid relid, AttrNumber attno);
+static Oid GetNewDictId(Relation relation, Oid indexId, AttrNumber dictIdColumn);
+
+/* ----------------------------------------------------------------
+ * Zstandard dictionary training related methods
+ * ----------------------------------------------------------------
+ */
+
+/*
+ * build_zstd_dictionary_internal
+ * 1) Validate that the given (relid, attno) can have a Zstd compression enabled on heap relation
+ * 2) Call the type-specific dictionary builder
+ * 3) Train a dictionary via ZDICT_trainFromBuffer()
+ * 4) Insert dictionary into pg_zstd_dictionaries
+ * 5) Update pg_attribute.attoptions with dictid
+ */
+pg_attribute_unused()
+static bool
+build_zstd_dictionary_internal(Oid relid, AttrNumber attno)
+{
+#ifdef USE_ZSTD
+ Relation catalogRel;
+ TupleDesc catTupDesc;
+ Oid dictid;
+ Relation rel;
+ TupleDesc tupleDesc;
+ Form_pg_attribute att;
+ AttributeOpts *attopt;
+ HeapTuple typeTup;
+ Form_pg_type typeForm;
+ Oid baseTypeOid;
+ Oid train_func;
+ Datum dictDatum;
+ ZstdTrainingData *dict;
+ char *samples_buffer;
+ size_t *sample_sizes;
+ size_t nitems;
+ uint32 dictionary_size;
+ void *dict_data;
+ size_t dict_size;
+
+ /* ----
+ * 1) Open user relation just to verify it's a normal table and has Zstd compression
+ * ----
+ */
+ rel = table_open(relid, AccessShareLock);
+ if (rel->rd_rel->relkind != RELKIND_RELATION)
+ {
+ table_close(rel, AccessShareLock);
+ return false; /* not a regular table */
+ }
+
+ /* If the column doesn't use Zstd, nothing to do */
+ tupleDesc = RelationGetDescr(rel);
+ att = TupleDescAttr(tupleDesc, attno - 1);
+ if (att->attcompression != TOAST_ZSTD_COMPRESSION)
+ {
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Check attoptions for user-requested dictionary size, etc. */
+ attopt = get_attribute_options(relid, attno);
+ if (attopt && attopt->zstd_dict_size == 0)
+ {
+ /* user explicitly says "no dictionary needed" */
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /*
+ * 2) Look up the type's custom dictionary builder function We'll call it
+ * to get sample data. Then we can close 'rel' because we don't need it
+ * open to do the actual Zdict training.
+ */
+ typeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typeTup))
+ {
+ table_close(rel, AccessShareLock);
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ }
+ typeForm = (Form_pg_type) GETSTRUCT(typeTup);
+
+ if (typeForm->typlen != -1)
+ {
+ ReleaseSysCache(typeTup);
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Get the base type */
+ baseTypeOid = get_element_type(typeForm->oid);
+ train_func = InvalidOid;
+
+ if (OidIsValid(baseTypeOid))
+ {
+ HeapTuple baseTypeTup;
+ Form_pg_type baseTypeForm;
+
+ /* It's an array type: get the base type's training function */
+ baseTypeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(baseTypeOid));
+ if (!HeapTupleIsValid(baseTypeTup))
+ ereport(ERROR,
+ (errmsg("Cache lookup failed for base type %u", baseTypeOid)));
+
+ baseTypeForm = (Form_pg_type) GETSTRUCT(baseTypeTup);
+ train_func = baseTypeForm->typebuildzstddictionary;
+ ReleaseSysCache(baseTypeTup);
+ }
+ else
+ train_func = typeForm->typebuildzstddictionary;
+
+ /* If the type does not supply a builder, skip */
+ if (!OidIsValid(train_func))
+ {
+ ReleaseSysCache(typeTup);
+ table_close(rel, AccessShareLock);
+ return false;
+ }
+
+ /* Call the type-specific builder. It should return ZstdTrainingData */
+ dictDatum = OidFunctionCall2(train_func,
+ PointerGetDatum(rel), /* pass relation ref */
+ PointerGetDatum(att));
+ ReleaseSysCache(typeTup);
+
+ /* We no longer need the user relation open */
+ table_close(rel, AccessShareLock);
+
+ dict = (ZstdTrainingData *) DatumGetPointer(dictDatum);
+ if (!dict || dict->nitems == 0)
+ return false;
+
+ /*
+ * 3) Train a Zstd dictionary in-memory.
+ */
+ samples_buffer = dict->sample_buffer;
+ sample_sizes = dict->sample_sizes;
+ nitems = dict->nitems;
+
+ dictionary_size = (!attopt ? DEFAULT_ZSTD_DICT_SIZE
+ : attopt->zstd_dict_size);
+
+ /* Allocate buffer for dictionary training result */
+ dict_data = palloc(dictionary_size);
+ dict_size = ZDICT_trainFromBuffer(dict_data,
+ dictionary_size,
+ samples_buffer,
+ sample_sizes,
+ nitems);
+ if (ZDICT_isError(dict_size))
+ {
+ elog(LOG, "Zstd dictionary training failed: %s",
+ ZDICT_getErrorName(dict_size));
+ pfree(dict_data);
+ return false;
+ }
+
+ /* Open the catalog relation with ShareRowExclusiveLock */
+ catalogRel = table_open(ZstdDictionariesRelationId, ShareRowExclusiveLock);
+ catTupDesc = RelationGetDescr(catalogRel);
+ dictid = GetNewDictId(catalogRel, ZstdDictidIndexId, Anum_pg_zstd_dictionaries_dictid);
+
+ /* Now copy that finalized dictionary into a bytea. */
+ {
+ /* We’ll store this bytea in pg_zstd_dictionaries. */
+ Datum values[Natts_pg_zstd_dictionaries];
+ bool nulls[Natts_pg_zstd_dictionaries];
+ HeapTuple tup;
+
+ bytea *dict_bytea = (bytea *) palloc(VARHDRSZ + dict_size);
+
+ SET_VARSIZE(dict_bytea, VARHDRSZ + dict_size);
+ memcpy(VARDATA(dict_bytea), dict_data, dict_size);
+
+ MemSet(values, 0, sizeof(values));
+ MemSet(nulls, false, sizeof(nulls));
+
+ values[Anum_pg_zstd_dictionaries_dictid - 1] = ObjectIdGetDatum(dictid);
+ values[Anum_pg_zstd_dictionaries_dict - 1] = PointerGetDatum(dict_bytea);
+
+ tup = heap_form_tuple(catTupDesc, values, nulls);
+ CatalogTupleInsert(catalogRel, tup);
+ heap_freetuple(tup);
+
+ pfree(dict_bytea);
+ }
+
+ pfree(dict_data);
+ pfree(samples_buffer);
+ pfree(sample_sizes);
+ pfree(dict);
+
+ /*
+ * 5) Update pg_attribute.attoptions with "dictid" => dictid so the column
+ * knows which dictionary to use at compression time.
+ */
+ {
+ Relation attRel = table_open(AttributeRelationId, RowExclusiveLock);
+ HeapTuple atttup,
+ newtuple;
+ Datum attoptionsDatum,
+ newOptions;
+ bool isnull;
+ Datum repl_val[Natts_pg_attribute];
+ bool repl_null[Natts_pg_attribute];
+ bool repl_repl[Natts_pg_attribute];
+ DefElem *def;
+
+ atttup = SearchSysCacheAttNum(relid, attno);
+ if (!HeapTupleIsValid(atttup))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_COLUMN),
+ errmsg("column number %d of relation \"%u\" does not exist",
+ attno, relid)));
+
+ /* Build new attoptions with dictid=... */
+ def = makeDefElem("dictid",
+ (Node *) makeString(psprintf("%u", dictid)),
+ -1);
+
+ attoptionsDatum = SysCacheGetAttr(ATTNUM, atttup,
+ Anum_pg_attribute_attoptions,
+ &isnull);
+ newOptions = transformRelOptions(isnull ? (Datum) 0 : attoptionsDatum,
+ list_make1(def),
+ NULL, NULL,
+ false, false);
+ /* Validate them (throws error if invalid) */
+ (void) attribute_reloptions(newOptions, true);
+
+ MemSet(repl_null, false, sizeof(repl_null));
+ MemSet(repl_repl, false, sizeof(repl_repl));
+
+ if (newOptions != (Datum) 0)
+ repl_val[Anum_pg_attribute_attoptions - 1] = newOptions;
+ else
+ repl_null[Anum_pg_attribute_attoptions - 1] = true;
+
+ repl_repl[Anum_pg_attribute_attoptions - 1] = true;
+
+ newtuple = heap_modify_tuple(atttup,
+ RelationGetDescr(attRel),
+ repl_val,
+ repl_null,
+ repl_repl);
+
+ CatalogTupleUpdate(attRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+
+ ReleaseSysCache(atttup);
+
+ table_close(attRel, NoLock);
+ }
+
+ /**
+ * Done inserting dictionary and updating attribute.
+ * Unlock the table (locks remain held until transaction commit)
+ */
+ table_close(catalogRel, NoLock);
+
+ return true;
+#else
+ return false;
+#endif
+}
+
+/*
+ * Acquire a new unique DictId for a relation.
+ *
+ * Assumes the relation is already locked with ShareRowExclusiveLock,
+ * ensuring that concurrent transactions cannot generate duplicate DictIds.
+ */
+pg_attribute_unused()
+static Oid
+GetNewDictId(Relation relation, Oid indexId, AttrNumber dictIdColumn)
+{
+ Relation indexRel = index_open(indexId, AccessShareLock);
+ Oid maxDictId = InvalidDictId;
+ SysScanDesc scan;
+ HeapTuple tuple;
+ bool collision;
+ ScanKeyData key;
+ Oid newDictId;
+
+ /* Retrieve the maximum existing DictId by scanning in reverse order */
+ scan = systable_beginscan_ordered(relation, indexRel, SnapshotAny, 0, NULL);
+ tuple = systable_getnext_ordered(scan, BackwardScanDirection);
+ if (HeapTupleIsValid(tuple))
+ {
+ Datum value;
+ bool isNull;
+
+ value = heap_getattr(tuple, dictIdColumn, RelationGetDescr(relation), &isNull);
+ if (!isNull)
+ maxDictId = DatumGetObjectId(value);
+ }
+ systable_endscan(scan);
+
+ newDictId = maxDictId + 1;
+ Assert(newDictId != InvalidDictId);
+
+ /* Check that the new DictId is indeed unique */
+ ScanKeyInit(&key,
+ dictIdColumn,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(newDictId));
+
+ scan = systable_beginscan(relation, indexRel->rd_id, true,
+ SnapshotAny, 1, &key);
+ collision = HeapTupleIsValid(systable_getnext(scan));
+ systable_endscan(scan);
+
+ if (collision)
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("unexpected collision for new DictId %d", newDictId)));
+
+ return newDictId;
+}
+
+/*
+ * get_zstd_dict - Fetches the ZSTD dictionary from the catalog
+ *
+ * dictid: The Oid of the dictionary to fetch.
+ *
+ * Returns: A pointer to a bytea containing the dictionary data.
+ */
+bytea *
+get_zstd_dict(Oid dictid)
+{
+ HeapTuple tuple;
+ Datum datum;
+ bool isNull;
+ bytea *dict_bytea;
+ bytea *result;
+ Size bytea_len;
+
+ /* Fetch the dictionary tuple from the syscache */
+ tuple = SearchSysCache1(ZSTDDICTIDOID, ObjectIdGetDatum(dictid));
+ if (!HeapTupleIsValid(tuple))
+ ereport(ERROR, (errmsg("Cache lookup failed for dictid %u", dictid)));
+
+ /* Get the dictionary attribute from the tuple */
+ datum = SysCacheGetAttr(ATTNUM, tuple, Anum_pg_zstd_dictionaries_dict, &isNull);
+ if (isNull)
+ ereport(ERROR, (errmsg("Dictionary not found for dictid %u", dictid)));
+
+ dict_bytea = DatumGetByteaP(datum);
+ if (dict_bytea == NULL)
+ ereport(ERROR, (errmsg("Failed to fetch dictionary")));
+
+ /* Determine the total size of the bytea (header + data) */
+ bytea_len = VARSIZE(dict_bytea);
+
+ result = palloc(bytea_len);
+ memcpy(result, dict_bytea, bytea_len);
+
+ /* Release the syscache tuple; the returned bytea is now independent */
+ ReleaseSysCache(tuple);
+
+ return result;
+}
+
+/*
+ * zstd_dictionary_builder
+ * Acquire samples from a column, store them in a SampleCollector,
+ * filter them, then build a ZstdTrainingData struct.
+ */
+Datum
+zstd_dictionary_builder(PG_FUNCTION_ARGS)
+{
+ ZstdTrainingData *dict = palloc0(sizeof(ZstdTrainingData));
+ Relation rel = (Relation) PG_GETARG_POINTER(0);
+ Form_pg_attribute att = (Form_pg_attribute) PG_GETARG_POINTER(1);
+ TupleDesc tupleDesc = RelationGetDescr(rel);
+
+ /* Acquire up to TARG_ROWS sample rows. */
+ HeapTuple *sample_rows = palloc(TARG_ROWS * sizeof(HeapTuple));
+ double totalrows = 0,
+ totaldeadrows = 0;
+ int num_sampled = acquire_sample_rows(rel, 0, sample_rows,
+ TARG_ROWS,
+ &totalrows,
+ &totaldeadrows);
+
+ /* Create a collector to accumulate raw varlena samples. */
+ size_t filtered_sample_count = 0;
+ size_t filtered_samples_size = 0;
+ char *samples_buffer;
+ size_t *sample_sizes;
+ size_t current_offset;
+ SampleCollector *collector;
+
+ if (num_sampled == 0)
+ {
+ pfree(sample_rows);
+ /* No samples were collected. */
+ PG_RETURN_POINTER(dict);
+ }
+
+ collector = palloc(sizeof(SampleCollector));
+ collector->samples = palloc(num_sampled * sizeof(SampleEntry));
+ collector->sample_count = 0;
+
+ /* Extract column data from each sampled row. */
+ for (int i = 0; i < num_sampled; i++)
+ {
+ bool isnull;
+ Datum value;
+
+ CHECK_FOR_INTERRUPTS();
+
+ value = heap_getattr(sample_rows[i],
+ att->attnum,
+ tupleDesc,
+ &isnull);
+ if (!isnull)
+ {
+ struct varlena *attr;
+ size_t size;
+ void *data;
+ SampleEntry entry;
+ int idx;
+
+ attr = (struct varlena *) PG_DETOAST_DATUM(value);
+ size = VARSIZE_ANY_EXHDR(attr);
+
+ if (filtered_samples_size + size > MaxAllocSize)
+ break;
+
+ data = palloc(size);
+ memcpy(data, VARDATA_ANY(attr), size);
+
+ entry.data = data;
+ entry.size = size;
+
+ idx = collector->sample_count;
+ collector->samples[idx] = entry;
+ collector->sample_count++;
+
+ filtered_samples_size += size;
+ filtered_sample_count++;
+ }
+ }
+
+ if (filtered_sample_count == 0)
+ {
+ pfree(sample_rows);
+ pfree(collector->samples);
+ pfree(collector);
+ /* No samples were collected, or they were too large. */
+ PG_RETURN_POINTER(dict);
+ }
+
+ /* Allocate a buffer for all sample data, plus an array of sample sizes. */
+ samples_buffer = palloc(filtered_samples_size);
+ sample_sizes = palloc(filtered_sample_count * sizeof(size_t));
+
+ /*
+ * Concatenate the samples into samples_buffer, recording each sample's
+ * size in sample_sizes.
+ */
+ current_offset = 0;
+ for (int i = 0; i < filtered_sample_count; i++)
+ {
+ memcpy(samples_buffer + current_offset,
+ collector->samples[i].data,
+ collector->samples[i].size);
+
+ sample_sizes[i] = collector->samples[i].size;
+ current_offset += collector->samples[i].size;
+
+ pfree(collector->samples[i].data);
+ }
+ pfree(sample_rows);
+ pfree(collector->samples);
+ pfree(collector);
+
+ dict->sample_buffer = samples_buffer;
+ dict->sample_sizes = sample_sizes;
+ dict->nitems = filtered_sample_count;
+
+ PG_RETURN_POINTER(dict);
+}
+
+Datum
+build_zstd_dict_for_attribute(PG_FUNCTION_ARGS)
+{
+#ifndef USE_ZSTD
+ PG_RETURN_BOOL(false);
+#else
+ text *tablename = PG_GETARG_TEXT_PP(0);
+ RangeVar *tablerel;
+ Oid tableoid = InvalidOid;
+ AttrNumber attno = PG_GETARG_INT32(1);
+ bool success;
+
+ /* Look up table name. */
+ tablerel = makeRangeVarFromNameList(textToQualifiedNameList(tablename));
+ tableoid = RangeVarGetRelid(tablerel, NoLock, false);
+ success = build_zstd_dictionary_internal(tableoid, attno);
+ PG_RETURN_BOOL(success);
+#endif
+}
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 2b5fbdcbd8..2b5500f45f 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -55,7 +55,7 @@
#include "utils/sortsupport.h"
#include "utils/syscache.h"
#include "utils/timestamp.h"
-
+#include "parser/analyze.h"
/* Per-index data for ANALYZE */
typedef struct AnlIndexData
@@ -85,9 +85,6 @@ static void compute_index_stats(Relation onerel, double totalrows,
MemoryContext col_context);
static VacAttrStats *examine_attribute(Relation onerel, int attnum,
Node *index_expr);
-static int acquire_sample_rows(Relation onerel, int elevel,
- HeapTuple *rows, int targrows,
- double *totalrows, double *totaldeadrows);
static int compare_rows(const void *a, const void *b, void *arg);
static int acquire_inherited_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
@@ -1195,7 +1192,7 @@ block_sampling_read_stream_next(ReadStream *stream,
* block. The previous sampling method put too much credence in the row
* density near the start of the table.
*/
-static int
+int
acquire_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
double *totalrows, double *totaldeadrows)
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index 3cb3ca1cca..c583e48167 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -95,6 +95,7 @@ typedef struct
bool updateTypmodout;
bool updateAnalyze;
bool updateSubscript;
+ bool updateGenerateDictionary;
/* New values for relevant attributes */
char storage;
Oid receiveOid;
@@ -103,6 +104,7 @@ typedef struct
Oid typmodoutOid;
Oid analyzeOid;
Oid subscriptOid;
+ Oid buildZstdDictionary;
} AlterTypeRecurseParams;
/* Potentially set by pg_upgrade_support functions */
@@ -122,6 +124,7 @@ static Oid findTypeSendFunction(List *procname, Oid typeOid);
static Oid findTypeTypmodinFunction(List *procname);
static Oid findTypeTypmodoutFunction(List *procname);
static Oid findTypeAnalyzeFunction(List *procname, Oid typeOid);
+static Oid findTypeGenerateDictionaryFunction(List *procname, Oid typeOid);
static Oid findTypeSubscriptingFunction(List *procname, Oid typeOid);
static Oid findRangeSubOpclass(List *opcname, Oid subtype);
static Oid findRangeCanonicalFunction(List *procname, Oid typeOid);
@@ -162,6 +165,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
List *typmodoutName = NIL;
List *analyzeName = NIL;
List *subscriptName = NIL;
+ List *generateDictionaryName = NIL;
char category = TYPCATEGORY_USER;
bool preferred = false;
char delimiter = DEFAULT_TYPDELIM;
@@ -190,6 +194,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
DefElem *alignmentEl = NULL;
DefElem *storageEl = NULL;
DefElem *collatableEl = NULL;
+ DefElem *generateDictionaryEl = NULL;
Oid inputOid;
Oid outputOid;
Oid receiveOid = InvalidOid;
@@ -198,6 +203,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
Oid typmodoutOid = InvalidOid;
Oid analyzeOid = InvalidOid;
Oid subscriptOid = InvalidOid;
+ Oid buildZstdDictionary = InvalidOid;
char *array_type;
Oid array_oid;
Oid typoid;
@@ -323,6 +329,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
defelp = &storageEl;
else if (strcmp(defel->defname, "collatable") == 0)
defelp = &collatableEl;
+ else if (strcmp(defel->defname, "build_zstd_dict") == 0)
+ defelp = &generateDictionaryEl;
else
{
/* WARNING, not ERROR, for historical backwards-compatibility */
@@ -455,6 +463,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
}
if (collatableEl)
collation = defGetBoolean(collatableEl) ? DEFAULT_COLLATION_OID : InvalidOid;
+ if (generateDictionaryEl)
+ generateDictionaryName = defGetQualifiedName(generateDictionaryEl);
/*
* make sure we have our required definitions
@@ -516,6 +526,15 @@ DefineType(ParseState *pstate, List *names, List *parameters)
errmsg("element type cannot be specified without a subscripting function")));
}
+ if (generateDictionaryName)
+ {
+ if (internalLength != -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type build_zstd_dict function must be specified only if data type is variable length.")));
+ buildZstdDictionary = findTypeGenerateDictionaryFunction(generateDictionaryName, typoid);
+ }
+
/*
* Check permissions on functions. We choose to require the creator/owner
* of a type to also own the underlying functions. Since creating a type
@@ -550,6 +569,9 @@ DefineType(ParseState *pstate, List *names, List *parameters)
if (analyzeOid && !object_ownercheck(ProcedureRelationId, analyzeOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(analyzeName));
+ if (buildZstdDictionary && !object_ownercheck(ProcedureRelationId, buildZstdDictionary, GetUserId()))
+ aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
+ NameListToString(generateDictionaryName));
if (subscriptOid && !object_ownercheck(ProcedureRelationId, subscriptOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(subscriptName));
@@ -601,7 +623,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array Dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ buildZstdDictionary); /* build_zstd_dict procedure */
Assert(typoid == address.objectId);
/*
@@ -643,7 +666,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ InvalidOid); /* build_zstd_dict procedure */
pfree(array_type);
@@ -706,6 +730,7 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
Oid receiveProcedure;
Oid sendProcedure;
Oid analyzeProcedure;
+ Oid buildZstdDictionary;
bool byValue;
char category;
char delimiter;
@@ -842,6 +867,9 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
/* Analysis function */
analyzeProcedure = baseType->typanalyze;
+ /* Generate dictionary function */
+ buildZstdDictionary = baseType->typebuildzstddictionary;
+
/*
* Domains don't need a subscript function, since they are not
* subscriptable on their own. If the base type is subscriptable, the
@@ -1078,7 +1106,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
basetypeMod, /* typeMod value */
typNDims, /* Array dimensions for base type */
typNotNull, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ buildZstdDictionary); /* build_zstd_dict procedure */
/*
* Create the array type that goes with it.
@@ -1119,7 +1148,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ InvalidOid); /* build_zstd_dict procedure */
pfree(domainArrayName);
@@ -1241,7 +1271,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
/* Enter the enum's values into pg_enum */
EnumValuesCreate(enumTypeAddr.objectId, stmt->vals);
@@ -1282,7 +1313,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
pfree(enumArrayName);
@@ -1583,7 +1615,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ InvalidOid); /* generate dictionary procedure - default */
Assert(typoid == InvalidOid || typoid == address.objectId);
typoid = address.objectId;
@@ -1650,7 +1683,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ InvalidOid); /* generate dictionary procedure - default */
Assert(multirangeOid == mltrngaddress.objectId);
/* Create the entry in pg_range */
@@ -1693,7 +1727,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ InvalidOid); /* generate dictionary procedure - default */
pfree(rangeArrayName);
@@ -1732,7 +1767,8 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ InvalidOid); /* generate dictionary procedure - default */
/* And create the constructor functions for this range type */
makeRangeConstructors(typeName, typeNamespace, typoid, rangeSubtype);
@@ -2257,6 +2293,31 @@ findTypeAnalyzeFunction(List *procname, Oid typeOid)
return procOid;
}
+static Oid
+findTypeGenerateDictionaryFunction(List *procname, Oid typeOid)
+{
+ Oid argList[2];
+ Oid procOid;
+
+ argList[0] = OIDOID;
+ argList[1] = INT4OID;
+
+ procOid = LookupFuncName(procname, 2, argList, true);
+ if (!OidIsValid(procOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_FUNCTION),
+ errmsg("function %s does not exist",
+ func_signature_string(procname, 1, NIL, argList))));
+
+ if (get_func_rettype(procOid) != INTERNALOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type build zstd dictionary function %s must return type %s",
+ NameListToString(procname), "internal")));
+
+ return procOid;
+}
+
static Oid
findTypeSubscriptingFunction(List *procname, Oid typeOid)
{
@@ -4440,6 +4501,19 @@ AlterType(AlterTypeStmt *stmt)
/* Replacing a subscript function requires superuser. */
requireSuper = true;
}
+ else if (strcmp(defel->defname, "build_zstd_dict") == 0)
+ {
+ if (defel->arg != NULL)
+ atparams.buildZstdDictionary =
+ findTypeGenerateDictionaryFunction(defGetQualifiedName(defel),
+ typeOid);
+ else
+ atparams.buildZstdDictionary = InvalidOid; /* NONE, remove function */
+
+ atparams.updateGenerateDictionary = true;
+ /* Replacing a canonical function requires superuser. */
+ requireSuper = true;
+ }
/*
* The rest of the options that CREATE accepts cannot be changed.
@@ -4602,6 +4676,11 @@ AlterTypeRecurse(Oid typeOid, bool isImplicitArray,
replaces[Anum_pg_type_typsubscript - 1] = true;
values[Anum_pg_type_typsubscript - 1] = ObjectIdGetDatum(atparams->subscriptOid);
}
+ if (atparams->updateGenerateDictionary)
+ {
+ replaces[Anum_pg_type_typebuildzstddictionary - 1] = true;
+ values[Anum_pg_type_typebuildzstddictionary - 1] = ObjectIdGetDatum(atparams->buildZstdDictionary);
+ }
newtup = heap_modify_tuple(tup, RelationGetDescr(catalog),
values, nulls, replaces);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index cdf185ea00..36f32f8590 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5280,6 +5280,9 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_ZSTD_COMPRESSION_ID:
+ result = "zstd";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index ad25cbb39c..e03ac8dddc 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -453,6 +453,9 @@ static const struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd", TOAST_ZSTD_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2d1de9c37b..47773e2919 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -731,7 +731,7 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate
#row_security = on
#default_table_access_method = 'heap'
#default_tablespace = '' # a tablespace name, '' uses the default
-#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
+#default_toast_compression = 'pglz' # 'pglz' or 'lz4' or 'zstd'
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#check_function_bodies = on
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 4f4ad2ee15..c2638fe4d8 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -8965,7 +8965,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
"a.attalign,\n"
"a.attislocal,\n"
"pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n"
- "array_to_string(a.attoptions, ', ') AS attoptions,\n"
+ "array_to_string(ARRAY(SELECT x FROM unnest(a.attoptions) AS x \n"
+ "WHERE x NOT LIKE 'dictid=%'), ', ') AS attoptions, \n"
"CASE WHEN a.attcollation <> t.typcollation "
"THEN a.attcollation ELSE 0 END AS attcollation,\n"
"pg_catalog.array_to_string(ARRAY("
@@ -11784,12 +11785,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
char *typmodout;
char *typanalyze;
char *typsubscript;
+ char *typebuildzstddictionary;
Oid typreceiveoid;
Oid typsendoid;
Oid typmodinoid;
Oid typmodoutoid;
Oid typanalyzeoid;
Oid typsubscriptoid;
+ Oid typebuildzstddictionaryoid;
char *typcategory;
char *typispreferred;
char *typdelim;
@@ -11822,10 +11825,18 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query,
"typsubscript, "
- "typsubscript::pg_catalog.oid AS typsubscriptoid ");
+ "typsubscript::pg_catalog.oid AS typsubscriptoid, ");
else
appendPQExpBufferStr(query,
- "'-' AS typsubscript, 0 AS typsubscriptoid ");
+ "'-' AS typsubscript, 0 AS typsubscriptoid, ");
+
+ if (fout->remoteVersion >= 180000)
+ appendPQExpBufferStr(query,
+ "typebuildzstddictionary, "
+ "typebuildzstddictionary::pg_catalog.oid AS typebuildzstddictionaryoid ");
+ else
+ appendPQExpBufferStr(query,
+ "'-' AS typebuildzstddictionary, 0 AS typebuildzstddictionaryoid ");
appendPQExpBufferStr(query, "FROM pg_catalog.pg_type "
"WHERE oid = $1");
@@ -11850,12 +11861,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
typmodout = PQgetvalue(res, 0, PQfnumber(res, "typmodout"));
typanalyze = PQgetvalue(res, 0, PQfnumber(res, "typanalyze"));
typsubscript = PQgetvalue(res, 0, PQfnumber(res, "typsubscript"));
+ typebuildzstddictionary = PQgetvalue(res, 0, PQfnumber(res, "typebuildzstddictionary"));
typreceiveoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typreceiveoid")));
typsendoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsendoid")));
typmodinoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodinoid")));
typmodoutoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodoutoid")));
typanalyzeoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typanalyzeoid")));
typsubscriptoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsubscriptoid")));
+ typebuildzstddictionaryoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typebuildzstddictionaryoid")));
typcategory = PQgetvalue(res, 0, PQfnumber(res, "typcategory"));
typispreferred = PQgetvalue(res, 0, PQfnumber(res, "typispreferred"));
typdelim = PQgetvalue(res, 0, PQfnumber(res, "typdelim"));
@@ -11911,7 +11924,8 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout);
if (OidIsValid(typanalyzeoid))
appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze);
-
+ if (OidIsValid(typebuildzstddictionaryoid))
+ appendPQExpBuffer(q, ",\n BUILD_ZSTD_DICT = %s", typebuildzstddictionary);
if (strcmp(typcollatable, "t") == 0)
appendPQExpBufferStr(q, ",\n COLLATABLE = true");
@@ -17170,6 +17184,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
case 'l':
cmname = "lz4";
break;
+ case 'z':
+ cmname = "zstd";
+ break;
default:
cmname = NULL;
break;
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index e6cf468ac9..0ba37bb175 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -2167,8 +2167,9 @@ describeOneTableDetails(const char *schemaname,
/* these strings are literal in our syntax, so not translated. */
printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
(compression[0] == 'l' ? "lz4" :
- (compression[0] == '\0' ? "" :
- "???"))),
+ (compression[0] == 'z' ? "zstd" :
+ (compression[0] == '\0' ? "" :
+ "???")))),
false, false);
}
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 13c4612cee..ddea1e0bcd 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -38,7 +38,8 @@ typedef enum ToastCompressionId
{
TOAST_PGLZ_COMPRESSION_ID = 0,
TOAST_LZ4_COMPRESSION_ID = 1,
- TOAST_INVALID_COMPRESSION_ID = 2,
+ TOAST_ZSTD_COMPRESSION_ID = 2,
+ TOAST_INVALID_COMPRESSION_ID = 3
} ToastCompressionId;
/*
@@ -48,10 +49,15 @@ typedef enum ToastCompressionId
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_COMPRESSION 'z'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
+#define InvalidDictId 0
+#define DEFAULT_ZSTD_LEVEL 3 /* Reffered from
+ * ZSTD_CLEVEL_DEFAULT */
+#define DEFAULT_ZSTD_DICT_SIZE (4 * 1024) /* 4 KB */
/* pglz compression/decompression routines */
extern struct varlena *pglz_compress_datum(const struct varlena *value);
@@ -65,6 +71,11 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* zstd compression/decompression routines */
+extern struct varlena *zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index e6ab8afffb..08bf3dfc67 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -33,6 +33,8 @@ typedef struct
int32 tai_size;
uint8 tai_colflags;
char tai_compression;
+ Oid dictid;
+ int zstd_level;
} ToastAttrInfo;
/*
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index 06ae8583c1..9ba6a1e64a 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -27,25 +27,54 @@ typedef struct toast_compress_header
* external size; see va_extinfo */
} toast_compress_header;
+typedef struct toast_compress_header_ext
+{
+ int32 vl_len_; /* varlena header (do not touch directly!) */
+ uint32 tcinfo; /* 2 bits for compression method and 30 bits
+ * external size; see va_extinfo */
+ uint32 ext_alg; /* compression method */
+ uint32 dictid; /* Dictionary Id */
+} toast_compress_header_ext;
+
+typedef struct CompressionInfo
+{
+ char cmethod;
+ Oid dictid;
+ int zstd_level; /* ZSTD compression level */
+} CompressionInfo;
+
/*
* Utilities for manipulation of header information for compressed
* toast entries.
*/
#define TOAST_COMPRESS_EXTSIZE(ptr) \
(((toast_compress_header *) (ptr))->tcinfo & VARLENA_EXTSIZE_MASK)
-#define TOAST_COMPRESS_METHOD(ptr) \
- (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
-
-#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
- do { \
- Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
- Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_compress_header *) (ptr))->tcinfo = \
- (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+#define TOAST_COMPRESS_METHOD(PTR) \
+ ( ((((toast_compress_header *) (PTR))->tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((toast_compress_header_ext *) (PTR))->ext_alg) \
+ : ( (((toast_compress_header *) (PTR))->tcinfo) >> VARLENA_EXTSIZE_BITS ) )
+
+#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method, dictid) \
+ do { \
+ Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
+ Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_ZSTD_COMPRESSION_ID); \
+ /* If the compression method is less than TOAST_ZSTD_COMPRESSION_ID, don't use ext_alg */ \
+ if ((cm_method) < TOAST_ZSTD_COMPRESSION_ID) { \
+ ((toast_compress_header *) (ptr))->tcinfo = \
+ (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+ } else { \
+ /* For compression methods after lz4, use 'VARLENA_EXTENDED_COMPRESSION_FLAG' \
+ in the top bits of tcinfo to indicate compression algorithm is stored in ext_alg. */ \
+ ((toast_compress_header_ext *) (ptr))->tcinfo = \
+ (len) | ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG << VARLENA_EXTSIZE_BITS); \
+ ((toast_compress_header_ext *) (ptr))->ext_alg = (cm_method); \
+ ((toast_compress_header_ext *) (ptr))->dictid = (dictid); \
+ } \
} while (0)
-extern Datum toast_compress_datum(Datum value, char cmethod);
+extern Datum toast_compress_datum(Datum value, CompressionInfo cmp);
extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile
index 2bbc7805fe..1ecd76dd31 100644
--- a/src/include/catalog/Makefile
+++ b/src/include/catalog/Makefile
@@ -81,7 +81,8 @@ CATALOG_HEADERS := \
pg_publication_namespace.h \
pg_publication_rel.h \
pg_subscription.h \
- pg_subscription_rel.h
+ pg_subscription_rel.h \
+ pg_zstd_dictionaries.h
GENERATED_HEADERS := $(CATALOG_HEADERS:%.h=%_d.h)
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index f427a89618..7cea56c2d5 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 202503071
+#define CATALOG_VERSION_NO 202503081
#endif
diff --git a/src/include/catalog/meson.build b/src/include/catalog/meson.build
index ec1cf467f6..e9cb6d911c 100644
--- a/src/include/catalog/meson.build
+++ b/src/include/catalog/meson.build
@@ -69,6 +69,7 @@ catalog_headers = [
'pg_publication_rel.h',
'pg_subscription.h',
'pg_subscription_rel.h',
+ 'pg_zstd_dictionaries.h',
]
# The .dat files we need can just be listed alphabetically.
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index cede992b6e..2bcae1eb52 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -12469,4 +12469,14 @@
proargtypes => 'int4',
prosrc => 'gist_stratnum_common' },
+# ZSTD generate dictionary training functions
+{ oid => '9241', descr => 'ZSTD generate dictionary support',
+ proname => 'zstd_dictionary_builder', prorettype => 'internal',
+ proargtypes => 'internal internal',
+ prosrc => 'zstd_dictionary_builder' },
+
+{ oid => '9242', descr => 'Build zstd dictionaries for a column.',
+ proname => 'build_zstd_dict_for_attribute', prorettype => 'bool',
+ proargtypes => 'text int4',
+ prosrc => 'build_zstd_dict_for_attribute' },
]
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index 6dca77e0a2..58a389a78c 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -40,7 +40,7 @@
descr => 'variable-length string, binary values escaped',
typname => 'bytea', typlen => '-1', typbyval => 'f', typcategory => 'U',
typinput => 'byteain', typoutput => 'byteaout', typreceive => 'bytearecv',
- typsend => 'byteasend', typalign => 'i', typstorage => 'x' },
+ typsend => 'byteasend', typalign => 'i', typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder' },
{ oid => '18', array_type_oid => '1002', descr => 'single character',
typname => 'char', typlen => '1', typbyval => 't', typcategory => 'Z',
typinput => 'charin', typoutput => 'charout', typreceive => 'charrecv',
@@ -83,7 +83,7 @@
typname => 'text', typlen => '-1', typbyval => 'f', typcategory => 'S',
typispreferred => 't', typinput => 'textin', typoutput => 'textout',
typreceive => 'textrecv', typsend => 'textsend', typalign => 'i',
- typstorage => 'x', typcollation => 'default' },
+ typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder', typcollation => 'default' },
{ oid => '26', array_type_oid => '1028',
descr => 'object identifier(oid), maximum 4 billion',
typname => 'oid', typlen => '4', typbyval => 't', typcategory => 'N',
@@ -446,7 +446,7 @@
typname => 'jsonb', typlen => '-1', typbyval => 'f', typcategory => 'U',
typsubscript => 'jsonb_subscript_handler', typinput => 'jsonb_in',
typoutput => 'jsonb_out', typreceive => 'jsonb_recv', typsend => 'jsonb_send',
- typalign => 'i', typstorage => 'x' },
+ typalign => 'i', typstorage => 'x', typebuildzstddictionary => 'zstd_dictionary_builder' },
{ oid => '4072', array_type_oid => '4073', descr => 'JSON path',
typname => 'jsonpath', typlen => '-1', typbyval => 'f', typcategory => 'U',
typinput => 'jsonpath_in', typoutput => 'jsonpath_out',
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index ff666711a5..bd82da8a88 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -227,6 +227,11 @@ CATALOG(pg_type,1247,TypeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(71,TypeRelati
*/
Oid typcollation BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_collation);
+ /*
+ * Custom generate dictionary procedure for the datatype (0 selects the
+ * default).
+ */
+ regproc typebuildzstddictionary BKI_DEFAULT(-) BKI_ARRAY_DEFAULT(-) BKI_LOOKUP_OPT(pg_proc);
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/*
@@ -380,7 +385,8 @@ extern ObjectAddress TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims,
bool typeNotNull,
- Oid typeCollation);
+ Oid typeCollation,
+ Oid generateDictionaryProcedure);
extern void GenerateTypeDependencies(HeapTuple typeTuple,
Relation typeCatalog,
diff --git a/src/include/catalog/pg_zstd_dictionaries.h b/src/include/catalog/pg_zstd_dictionaries.h
new file mode 100644
index 0000000000..5b0b729283
--- /dev/null
+++ b/src/include/catalog/pg_zstd_dictionaries.h
@@ -0,0 +1,53 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_zstd_dictionaries.h
+ * definition of the "zstd dictionay" system catalog (pg_zstd_dictionaries)
+ *
+ * src/include/catalog/pg_zstd_dictionaries.h
+ *
+ * NOTES
+ * The Catalog.pm module reads this file and derives schema
+ * information.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_ZSTD_DICTIONARIES_H
+#define PG_ZSTD_DICTIONARIES_H
+
+#include "catalog/genbki.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+
+/* ----------------
+ * pg_zstd_dictionaries definition. cpp turns this into
+ * typedef struct FormData_pg_zstd_dictionaries
+ * ----------------
+ */
+CATALOG(pg_zstd_dictionaries,9946,ZstdDictionariesRelationId)
+{
+ Oid dictid BKI_FORCE_NOT_NULL;
+
+ /*
+ * variable-length fields start here, but we allow direct access to dict
+ */
+ bytea dict BKI_FORCE_NOT_NULL;
+} FormData_pg_zstd_dictionaries;
+
+/* Pointer type to a tuple with the format of pg_zstd_dictionaries relation */
+typedef FormData_pg_zstd_dictionaries *Form_pg_zstd_dictionaries;
+
+DECLARE_TOAST(pg_zstd_dictionaries, 9947, 9948);
+
+DECLARE_UNIQUE_INDEX_PKEY(pg_zstd_dictionaries_dictid_index, 9949, ZstdDictidIndexId, pg_zstd_dictionaries, btree(dictid oid_ops));
+
+MAKE_SYSCACHE(ZSTDDICTIDOID, pg_zstd_dictionaries_dictid_index, 128);
+
+typedef struct ZstdTrainingData
+{
+ char *sample_buffer; /* Pointer to the raw sample buffer */
+ size_t *sample_sizes; /* Array of sample sizes */
+ size_t nitems; /* Number of sample sizes */
+} ZstdTrainingData;
+
+extern bytea *get_zstd_dict(Oid dictid);
+
+#endif /* PG_ZSTD_DICTIONARIES_H */
diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h
index f1bd18c49f..e494436870 100644
--- a/src/include/parser/analyze.h
+++ b/src/include/parser/analyze.h
@@ -17,6 +17,7 @@
#include "nodes/params.h"
#include "nodes/queryjumble.h"
#include "parser/parse_node.h"
+#include "access/htup.h"
/* Hook for plugins to get control at end of parse analysis */
typedef void (*post_parse_analyze_hook_type) (ParseState *pstate,
@@ -64,4 +65,8 @@ extern List *BuildOnConflictExcludedTargetlist(Relation targetrel,
extern SortGroupClause *makeSortGroupClauseForSetOp(Oid rescoltype, bool require_hash);
+extern int acquire_sample_rows(Relation onerel, int elevel,
+ HeapTuple *rows, int targrows,
+ double *totalrows, double *totaldeadrows);
+
#endif /* ANALYZE_H */
diff --git a/src/include/utils/attoptcache.h b/src/include/utils/attoptcache.h
index f684a772af..55a6ac6167 100644
--- a/src/include/utils/attoptcache.h
+++ b/src/include/utils/attoptcache.h
@@ -21,6 +21,12 @@ typedef struct AttributeOpts
int32 vl_len_; /* varlena header (do not touch directly!) */
float8 n_distinct;
float8 n_distinct_inherited;
+ double dictid; /* Oid is a 32-bit unsigned integer, but
+ * relopt_int is limited to INT_MAX, so it
+ * cannot represent the full range of Oid
+ * values. */
+ int zstd_dict_size;
+ int zstd_level;
} AttributeOpts;
extern AttributeOpts *get_attribute_options(Oid attrelid, int attnum);
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 2e8564d499..5fb0ab8beb 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -42,8 +42,9 @@ typedef struct varatt_external
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
*/
-#define VARLENA_EXTSIZE_BITS 30
-#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTSIZE_BITS 30
+#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTENDED_COMPRESSION_FLAG 0x3
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
@@ -122,6 +123,14 @@ typedef union
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
+ struct
+ {
+ uint32 va_header;
+ uint32 va_tcinfo;
+ uint32 va_cmp_alg;
+ uint32 va_cmp_dictid;
+ char va_data[FLEXIBLE_ARRAY_MEMBER];
+ } va_compressed_ext;
} varattrib_4b;
typedef struct
@@ -242,7 +251,14 @@ typedef struct
#endif /* WORDS_BIGENDIAN */
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
-#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
+/*
+ * If va_tcinfo >> VARLENA_EXTSIZE_BITS == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * use va_compressed_ext; otherwise, use the va_compressed.
+ */
+#define VARDATA_4B_C(PTR) \
+( (((varattrib_4b *)(PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG \
+ ? ((varattrib_4b *)(PTR))->va_compressed_ext.va_data \
+ : ((varattrib_4b *)(PTR))->va_compressed.va_data )
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
@@ -252,6 +268,7 @@ typedef struct
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
+#define VARHDRSZ_COMPRESSED_EXT offsetof(varattrib_4b, va_compressed_ext.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
@@ -327,8 +344,20 @@ typedef struct
/* Decompressed size and compression method of a compressed-in-line Datum */
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
-#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
- (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
+/*
+ * - "Extended" format is indicated by (va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * - For the non-extended formats, the method code is stored in the top bits of va_tcinfo.
+ * - In the extended format, the method code is stored in va_cmp_alg instead.
+ */
+#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
+( ((((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((varattrib_4b *) (PTR))->va_compressed_ext.va_cmp_alg) \
+ : ( (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo) >> VARLENA_EXTSIZE_BITS))
+
+#define VARDATA_COMPRESSED_GET_DICTID(PTR) \
+ ( ((((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((varattrib_4b *) (PTR))->va_compressed_ext.va_cmp_dictid) \
+ : InvalidDictId)
/* Same for external Datums; but note argument is a struct varatt_external */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
@@ -336,13 +365,27 @@ typedef struct
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
-#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
- do { \
- Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
- } while (0)
+#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
+ do { \
+ /* If desired, keep or expand the Assert checks for known methods: */ \
+ Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == TOAST_ZSTD_COMPRESSION_ID); \
+ if ((cm) < TOAST_ZSTD_COMPRESSION_ID) \
+ { \
+ /* Store the actual method in va_extinfo */ \
+ (toast_pointer).va_extinfo = (uint32)(len) \
+ | ((uint32)(cm) << VARLENA_EXTSIZE_BITS); \
+ } \
+ else \
+ { \
+ /* Store VARLENA_EXTENDED_COMPRESSION_FLAG in the top bits, \
+ meaning "extended" method. */ \
+ (toast_pointer).va_extinfo = (uint32)(len) | \
+ ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG \
+ << VARLENA_EXTSIZE_BITS); \
+ } \
+ } while (0)
/*
* Testing whether an externally-stored value is compressed now requires
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 4dd9ee7200..94495388ad 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -238,10 +238,11 @@ NOTICE: merging multiple inherited definitions of column "f1"
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
-- test alter compression method
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 7bd7642b4b..0ce4915217 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -233,6 +233,9 @@ HINT: Available values: pglz.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
HINT: Available values: pglz.
+SET default_toast_compression = 'zstd';
+ERROR: invalid value for parameter "default_toast_compression": "zstd"
+HINT: Available values: pglz.
SET default_toast_compression = 'lz4';
ERROR: invalid value for parameter "default_toast_compression": "lz4"
HINT: Available values: pglz.
diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out
new file mode 100644
index 0000000000..7de110a90a
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd.out
@@ -0,0 +1,123 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+ERROR: compression method zstd not supported
+DETAIL: This functionality requires the server to be built with zstd support.
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 10...
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1)
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 3: FROM cmdata_zstd
+ ^
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPR...
+ ^
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+ERROR: table "cmdata_zstd_2" does not exist
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: SELECT f1 FROM cmdata_zstd;
+ ^
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ERROR: relation "compressmv_zstd" does not exist
+LINE 2: FROM compressmv_zstd;
+ ^
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: UPDATE cmdata_zstd
+ ^
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+ERROR: materialized view "compressmv_zstd" does not exist
+DROP TABLE cmdata_zstd;
+ERROR: table "cmdata_zstd" does not exist
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out
new file mode 100644
index 0000000000..a540c99b37
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd_1.out
@@ -0,0 +1,181 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+ Table "public.cmdata_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ compression_method | row_count
+--------------------+-----------
+ zstd | 20
+(1 row)
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ data_slice
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+ fceea167a5a36dedd4bea2543c9f0f895fb98ab9159f51fd02
+(20 rows)
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+ Table "public.cmdata_zstd_2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+DROP TABLE cmdata_zstd_2;
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+ Materialized view "public.compressmv_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | | |
+View definition:
+ SELECT f1
+ FROM cmdata_zstd;
+
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ mv_compression
+----------------
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+(20 rows)
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ preview
+---------
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+(20 rows)
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out
index 215eb899be..ac5da3f5ab 100644
--- a/src/test/regress/expected/oidjoins.out
+++ b/src/test/regress/expected/oidjoins.out
@@ -71,6 +71,7 @@ NOTICE: checking pg_type {typmodout} => pg_proc {oid}
NOTICE: checking pg_type {typanalyze} => pg_proc {oid}
NOTICE: checking pg_type {typbasetype} => pg_type {oid}
NOTICE: checking pg_type {typcollation} => pg_collation {oid}
+NOTICE: checking pg_type {typebuildzstddictionary} => pg_proc {oid}
NOTICE: checking pg_attribute {attrelid} => pg_class {oid}
NOTICE: checking pg_attribute {atttypid} => pg_type {oid}
NOTICE: checking pg_attribute {attcollation} => pg_collation {oid}
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 37b6d21e1f..407a0644f8 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -119,7 +119,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# The stats test resets stats, so nothing else needing stats access can be in
# this group.
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats predicate
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd memoize stats predicate
# event_trigger depends on create_am and cannot run concurrently with
# any test that runs DDL
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index 490595fcfb..e29909558f 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -102,6 +102,7 @@ CREATE TABLE cminh() INHERITS (cmdata, cmdata3);
-- test default_toast_compression GUC
SET default_toast_compression = '';
SET default_toast_compression = 'I do not exist compression';
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql
new file mode 100644
index 0000000000..7cf93e3de2
--- /dev/null
+++ b/src/test/regress/sql/compression_zstd.sql
@@ -0,0 +1,97 @@
+\set HIDE_TOAST_COMPRESSION false
+
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+
+-- Create a helper function to generate extra-large values.
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+$$
+ SELECT string_agg(md5(g::text), '')
+ FROM generate_series(1,256) g
+$$;
+
+-- Insert 5 extra-large rows to force externally stored compression.
+DO $$
+BEGIN
+ FOR i IN 1..5 LOOP
+ INSERT INTO cmdata_zstd (f1)
+ VALUES (large_val() || repeat('a', 4000));
+ END LOOP;
+END $$;
+
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9840060997..adb94ee7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -886,6 +886,7 @@ FormData_pg_ts_parser
FormData_pg_ts_template
FormData_pg_type
FormData_pg_user_mapping
+FormData_pg_zstd_dictionaries
FormExtraData_pg_attribute
Form_pg_aggregate
Form_pg_am
@@ -945,6 +946,7 @@ Form_pg_ts_parser
Form_pg_ts_template
Form_pg_type
Form_pg_user_mapping
+Form_pg_zstd_dictionaries
FormatNode
FreeBlockNumberArray
FreeListData
@@ -2582,6 +2584,8 @@ STARTUPINFO
STRLEN
SV
SYNCHRONIZATION_BARRIER
+SampleCollector
+SampleEntry
SampleScan
SampleScanGetSampleSize_function
SampleScanState
@@ -3306,6 +3310,7 @@ ZSTD_cParameter
ZSTD_inBuffer
ZSTD_outBuffer
ZstdCompressorState
+ZstdTrainingData
_SPI_connection
_SPI_plan
__m128i
base-commit: 8021c77769e90cc804121d61a1bb7bcc4652d48b
--
2.47.1
[image/png] image.png (248.3K, ../../CAFAfj_FbdeabZEqU_vCCLar07DvF0SJRgimW8A7ZmD=UPwE6VA@mail.gmail.com/3-image.png)
download | view image
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-03-17 20:02 Robert Haas <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
1 sibling, 1 reply; 21+ messages in thread
From: Robert Haas @ 2025-03-17 20:02 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: Aleksander Alekseev <[email protected]>; pgsql-hackers; Tom Lane <[email protected]>
On Fri, Mar 7, 2025 at 8:36 PM Nikhil Kumar Veldanda
<[email protected]> wrote:
> struct /* Extended compression format */
> {
> uint32 va_header;
> uint32 va_tcinfo;
> uint32 va_cmp_alg;
> uint32 va_cmp_dictid;
> char va_data[FLEXIBLE_ARRAY_MEMBER];
> } va_compressed_ext;
> } varattrib_4b;
First, thanks for sending along the performance results. I agree that
those are promising. Second, thanks for sending these design details.
The idea of keeping dictionaries in pg_zstd_dictionaries literally
forever doesn't seem very appealing, but I'm not sure what the other
options are. I think we've established in previous work in this area
that compressed values can creep into unrelated tables and inside
records or other container types like ranges. Therefore, we have no
good way of knowing when a dictionary is unreferenced and can be
dropped. So in that sense your decision to keep them forever is
"right," but it's still unpleasant. It would even be necessary to make
pg_upgrade carry them over to new versions.
If we could make sure that compressed datums never leaked out into
other tables, then tables could depend on dictionaries and
dictionaries could be dropped when there were no longer any tables
depending on them. But like I say, previous work suggested that this
would be very difficult to achieve. However, without that, I imagine
users generating new dictionaries regularly as the data changes and
eventually getting frustrated that they can't get rid of the old ones.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-15 18:13 Nikhil Kumar Veldanda <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-04-15 18:13 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: pgsql-hackers
Hi Robert,
Thank you for your response, and apologies for the delay in getting
back to you. You raised some important concerns in your reply, I’ve
worked hard to understand and hopefully address these two:
* Dictionary Cleanup via Dependency Tracking
* Addressing Compressed Datum Leaks problem (via CTAS, INSERT INTO ...
SELECT ...)
Dictionary Cleanup via Dependency Tracking:
To address your question on how we can safely clean up unused
dictionaries, I’ve implemented a mechanism based on PostgreSQL’s
standard dependency system (pg_depend), permit me to explain.
When a Zstandard dictionary is created for a table, we record a
DEPENDENCY_NORMAL dependency from the table to the dictionary. This
ensures that when the table is dropped, the corresponding entry is
removed from the pg_depend catalog. Users can then call the
cleanup_unused_dictionaries() function to remove any dictionaries that
are no longer referenced by any table.
// create dependency,
{
ObjectAddress dictObj;
ObjectAddress relation;
ObjectAddressSet(dictObj, ZstdDictionariesRelationId, dictid);
ObjectAddressSet(relation, RelationRelationId, relid);
/* NORMAL dependency: relid → Dictionary */
recordDependencyOn(&relation, &dictObj, DEPENDENCY_NORMAL);
}
Example: Consider two tables, each using its own Zstandard dictionary:
test=# \dt+
List of tables
Schema | Name | Type | Owner | Persistence | Access method |
Size | Description
--------+-------+-------+----------+-------------+---------------+-------+-------------
public | temp | table | nikhilkv | permanent | heap | 16 kB |
public | temp1 | table | nikhilkv | permanent | heap | 16 kB |
(2 rows)
// Dictionary dependencies
test=# select * from pg_depend where refclassid = 9946;
classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype
---------+-------+----------+------------+----------+-------------+---------
1259 | 16389 | 0 | 9946 | 1 | 0 | n
1259 | 16394 | 0 | 9946 | 2 | 0 | n
(2 rows)
// the corresponding dictionaries:
test=# select * from pg_zstd_dictionaries ;
dictid |
dict
--------+----------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------
--------------------------------------
1 | \x37a430ec71451a10091010df303333b3770a33f1783c1e8fc7e3f1783ccff3bcf7d442414141414141414141414141414141414141414
14141414141a15028140a8542a15028140a85a2288aa2284a297d74e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1f1783c1e8fc7e3f1789ee779ef01
0100000004000000080000004c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e73656374657475722061646970697363696
e6720656c69742e204c6f72656d2069
2 | \x37a430ec7d1a933a091010df303333b3770a33f1783c1e8fc7e3f1783ccff3bcf7d442414141414141414141414141414141414141414
14141414141a15028140a8542a15028140a85a2288aa2284a297d74e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1f1783c1e8fc7e3f1789ee779ef01
0100000004000000080000004e696b68696c206b756d616e722076656c64616e64612c206973206f6b61792063616e6469646174652c2068652069732
0696e2073656174746c65204e696b68696c20
(2 rows)
If cleanup_unused_dictionaries() is called while the dependencies
still exist, nothing is removed:
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
0
(1 row)
After dropping temp1, the associated dictionary becomes eligible for cleanup:
test=# drop table temp1;
DROP TABLE
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
1
(1 row)
________________________________
Addressing Compressed Datum Leaks problem (via CTAS, INSERT INTO ... SELECT ...)
As compressed datums can be copied to other unrelated tables via CTAS,
INSERT INTO ... SELECT, or CREATE TABLE ... EXECUTE, I’ve introduced a
method inheritZstdDictionaryDependencies. This method is invoked at
the end of such statements and ensures that any dictionary
dependencies from source tables are copied to the destination table.
We determine the set of source tables using the relationOids field in
PlannedStmt.
This guarantees that if compressed datums reference a zstd dictionary
the destination table is marked as dependent on the dictionaries that
the source tables depend on, preventing premature cleanup by
cleanup_unused_dictionaries.
Example: Consider this example where we have two tables which has
their own dictionary
List of tables
Schema | Name | Type | Owner | Persistence | Access method |
Size | Description
--------+-------+-------+----------+-------------+---------------+-------+-------------
public | temp | table | nikhilkv | permanent | heap | 16 kB |
public | temp1 | table | nikhilkv | permanent | heap | 16 kB |
(2 rows)
Using CTAS (CREATE TABLE AS), one table is copied to another. In this
case, the compressed datums in the temp table are copied to copy_tbl.
Since the dictionary is shared between two tables, a dependency on
that dictionary is also established for the destination table. Even if
the original temp table is deleted and cleanup is triggered, the
dictionary will not be dropped because there remains an active
dependency.
test=# create table copy_tbl as select * from temp;
SELECT 20
// dictid 1 is shared between two tables.
test=# select * from pg_depend where refclassid = 9946;
classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype
---------+-------+----------+------------+----------+-------------+---------
1259 | 16389 | 0 | 9946 | 1 | 0 | n
1259 | 16404 | 0 | 9946 | 1 | 0 | n
1259 | 16399 | 0 | 9946 | 3 | 0 | n
(3 rows)
// After dropping the temp tale where dictid 1 is used to compress datums
test=# drop table temp;
DROP TABLE
// dependency for temp table is dropped.
test=# select * from pg_depend where refclassid = 9946;
classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype
---------+-------+----------+------------+----------+-------------+---------
1259 | 16404 | 0 | 9946 | 1 | 0 | n
1259 | 16399 | 0 | 9946 | 3 | 0 | n
(2 rows)
// No dictionaries are being deleted.
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
0
(1 row)
Once the new copy_tbl is also deleted, the dictionary can be dropped
because no dependency exists on it:
test=# drop table copy_tbl;
DROP TABLE
// The dictionary is then deleted.
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
1
(1 row)
Another example using composite types, including a more complex
scenario involving two source tables.
// Create a base composite type with two text fields
test=# create type my_composite as (f1 text, f2 text);
CREATE TYPE
// Create a nested composite type that uses my_composite twice
test=# create type my_composite1 as (f1 my_composite, f2 my_composite);
CREATE TYPE
test=# \d my_composite
Composite type "public.my_composite"
Column | Type | Collation | Nullable | Default
--------+------+-----------+----------+---------
f1 | text | | |
f2 | text | | |
test=# \d my_composite1
Composite type "public.my_composite1"
Column | Type | Collation | Nullable | Default
--------+--------------+-----------+----------+---------
f1 | my_composite | | |
f2 | my_composite | | |
// Sample table with ZSTD dictionary compression on text columns
test=# \d+ orders
Table "public.orders"
Column | Type | Collation | Nullable | Default | Storage |
Compression | Stats target | Description
-------------+---------+-----------+----------+---------+----------+-------------+--------------+-------------
order_id | integer | | | | plain |
| |
customer_id | integer | | | | plain |
| |
random1 | text | | | | extended |
zstd | |
random2 | text | | | | extended |
zstd | |
Access method: heap
// Sample table with ZSTD dictionary compression on one of the text column
test=# \d+ customers
Table "public.customers"
Column | Type | Collation | Nullable | Default | Storage |
Compression | Stats target | Description
-------------+---------+-----------+----------+---------+----------+-------------+--------------+-------------
customer_id | integer | | | | plain |
| |
random3 | text | | | | extended |
zstd | |
random4 | text | | | | extended |
| |
Access method: heap
// Check existing dictionaries: dictid 1 for random1, dictid 2 for
random2, dictid 3 for random3 attribute
test=# select dictid from pg_zstd_dictionaries;
dictid
--------
1
2
3
(3 rows)
// List all objects dependent on ZSTD dictionaries
test=# select objid::regclass, * from pg_depend where refclassid = 9946;
objid | classid | objid | objsubid | refclassid | refobjid |
refobjsubid | deptype
-----------+---------+-------+----------+------------+----------+-------------+---------
orders | 1259 | 16391 | 0 | 9946 | 1 |
0 | n
orders | 1259 | 16391 | 0 | 9946 | 2 |
0 | n
customers | 1259 | 16396 | 0 | 9946 | 3 |
0 | n
(3 rows)
// Create new table using nested composite type
// This copies compressed datums into temp1.
test=# create table temp1 as
select ROW(
ROW(random3, random4)::my_composite,
ROW(random1, random2)::my_composite
)::my_composite1
from customers full outer join orders using (customer_id);
SELECT 51
test=# select objid::regclass, * from pg_depend where refclassid = 9946;
objid | classid | objid | objsubid | refclassid | refobjid |
refobjsubid | deptype
-----------+---------+-------+----------+------------+----------+-------------+---------
orders | 1259 | 16391 | 0 | 9946 | 1 |
0 | n
temp1 | 1259 | 16423 | 0 | 9946 | 1 |
0 | n
orders | 1259 | 16391 | 0 | 9946 | 2 |
0 | n
temp1 | 1259 | 16423 | 0 | 9946 | 2 |
0 | n
temp1 | 1259 | 16423 | 0 | 9946 | 3 |
0 | n
customers | 1259 | 16396 | 0 | 9946 | 3 |
0 | n
(6 rows)
// Drop the original source tables.
test=# drop table orders;
DROP TABLE
test=# drop table customers ;
DROP TABLE
// Even after dropping orders, customers table, temp1 still holds
references to the dictionaries.
test=# select objid::regclass, * from pg_depend where refclassid = 9946;
objid | classid | objid | objsubid | refclassid | refobjid |
refobjsubid | deptype
-------+---------+-------+----------+------------+----------+-------------+---------
temp1 | 1259 | 16423 | 0 | 9946 | 1 | 0 | n
temp1 | 1259 | 16423 | 0 | 9946 | 2 | 0 | n
temp1 | 1259 | 16423 | 0 | 9946 | 3 | 0 | n
(3 rows)
// Attempt cleanup, No cleanup occurs, because temp1 table still
depends on the dictionaries.
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
0
(1 row)
test=# select dictid from pg_zstd_dictionaries ;
dictid
--------
1
2
3
(3 rows)
// Drop the destination table
test=# drop table temp1;
DROP TABLE
// Confirm no remaining dependencies
test=# select objid::regclass, * from pg_depend where refclassid = 9946;
objid | classid | objid | objsubid | refclassid | refobjid |
refobjsubid | deptype
-------+---------+-------+----------+------------+----------+-------------+---------
(0 rows)
// Cleanup now succeeds
test=# select cleanup_unused_dictionaries();
cleanup_unused_dictionaries
-----------------------------
3
(1 row)
test=# select dictid from pg_zstd_dictionaries ;
dictid
--------
(0 rows)
This design ensures that:
Dictionaries are only deleted when no table depends on them.
We avoid costly decompression/recompression to avoid compressed datum leakage.
We don’t retain dictionaries forever.
These changes are the core additions in this revision of the patch to
address concern around long-lived dictionaries and compressed datum
leakage. Additionally, this update incorporates feedback by enabling
automatic zstd dictionary generation and cleanup during the VACUUM
process and includes changes to support copying ZSTD dictionaries
during pg_upgrade.
Patch summary:
v11-0001-varattrib_4b-changes-and-macros-update-needed-to.patch
Refactors varattrib_4b structures and updates related macros to enable
ZSTD dictionary support.
v11-0002-Zstd-compression-and-decompression-routines-incl.patch
Adds ZSTD compression and decompression routines, and introduces a new
catalog to store dictionary metadata.
v11-0003-Zstd-dictionary-training-process.patch
Implements the dictionary training workflow. Includes built-in support
for text and jsonb types. Allows users to define custom sampling
functions per type by specifying a C function name in the
pg_type.typzstdsampling field.
v11-0004-Dependency-tracking-mechanism-to-track-compresse.patch
Introduces a dependency tracking mechanism using pg_depend to record
which ZSTD dictionaries a table depends on. When compressed datums
that rely on a dictionary are copied to unrelated target tables, the
corresponding dictionary dependencies from the source table are also
recorded for the target table, ensuring the dictionaries are not
prematurely cleaned up.
v11-0005-generate-and-cleanup-dictionaries-using-vacuum.patch
Adds integration with VACUUM to automatically generate and clean up
ZSTD dictionaries.
v11-0006-pg_dump-pg_upgrade-needed-changes-to-support-new.patch
Extends pg_dump and pg_upgrade to support migrating ZSTD dictionaries
and their dependencies during pg_upgrade.
v11-0007-Some-tests-related-to-zstd-dictionary-based-comp.patch
Provides test coverage for ZSTD dictionary-based compression features,
including training, usage, and cleanup.
I hope that these changes address your concerns, any thoughts or
suggestions on this approach are welcome.
Best regards,
Nikhil Veldanda
On Mon, Mar 17, 2025 at 1:03 PM Robert Haas <[email protected]> wrote:
>
> On Fri, Mar 7, 2025 at 8:36 PM Nikhil Kumar Veldanda
> <[email protected]> wrote:
> > struct /* Extended compression format */
> > {
> > uint32 va_header;
> > uint32 va_tcinfo;
> > uint32 va_cmp_alg;
> > uint32 va_cmp_dictid;
> > char va_data[FLEXIBLE_ARRAY_MEMBER];
> > } va_compressed_ext;
> > } varattrib_4b;
>
> First, thanks for sending along the performance results. I agree that
> those are promising. Second, thanks for sending these design details.
>
> The idea of keeping dictionaries in pg_zstd_dictionaries literally
> forever doesn't seem very appealing, but I'm not sure what the other
> options are. I think we've established in previous work in this area
> that compressed values can creep into unrelated tables and inside
> records or other container types like ranges. Therefore, we have no
> good way of knowing when a dictionary is unreferenced and can be
> dropped. So in that sense your decision to keep them forever is
> "right," but it's still unpleasant. It would even be necessary to make
> pg_upgrade carry them over to new versions.
>
> If we could make sure that compressed datums never leaked out into
> other tables, then tables could depend on dictionaries and
> dictionaries could be dropped when there were no longer any tables
> depending on them. But like I say, previous work suggested that this
> would be very difficult to achieve. However, without that, I imagine
> users generating new dictionaries regularly as the data changes and
> eventually getting frustrated that they can't get rid of the old ones.
>
> --
> Robert Haas
> EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v11-0005-generate-and-cleanup-dictionaries-using-vacuum.patch (3.7K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/2-v11-0005-generate-and-cleanup-dictionaries-using-vacuum.patch)
download | inline diff:
From ed65970ccd7953aebc86111333f55f03260885e1 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:51:43 +0000
Subject: [PATCH v11 5/7] generate and cleanup dictionaries using vacuum
---
src/backend/catalog/pg_zstd_dictionaries.c | 56 ++++++++++++++++++++++
src/backend/commands/vacuum.c | 4 ++
src/include/catalog/pg_zstd_dictionaries.h | 1 +
3 files changed, 61 insertions(+)
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
index 5ae8ed71e48..08a6883ecd4 100644
--- a/src/backend/catalog/pg_zstd_dictionaries.c
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -568,6 +568,62 @@ cleanup_unused_zstd_dictionaries_internal(void)
return dropped_count;
}
+/*
+ * generate_or_cleanup_zstd_dictionaries_for_relation
+ *
+ * Opens the relation identified by relid, iterates over its attributes,
+ * and for each valid (non-dropped, user-defined) attribute, calls
+ * build_zstd_dictionary_internal.
+ */
+void
+generate_or_cleanup_zstd_dictionaries_for_relation(Oid relid)
+{
+ Relation rel;
+ TupleDesc tupdesc;
+
+ /* Start a new transaction */
+ StartTransactionCommand();
+
+ /* Push an active snapshot toast want snapshot */
+ PushActiveSnapshot(GetTransactionSnapshot());
+
+ /* Open the relation using table_open (or relation_open) */
+ rel = table_open(relid, AccessShareLock);
+ tupdesc = RelationGetDescr(rel);
+
+ /* Iterate over all attributes of the relation */
+ for (int i = 0; i < tupdesc->natts; i++)
+ {
+ Form_pg_attribute attr = TupleDescAttr(tupdesc, i);
+
+ /* Skip dropped attributes and system columns (attnum <= 0) */
+ if (attr->attisdropped || attr->attnum <= 0)
+ continue;
+
+ /* Call your dictionary-building function for this attribute */
+ build_zstd_dictionary_internal(relid, attr->attnum);
+
+ /*
+ * If build_zstd_dictionary_internal performs modifications that
+ * subsequent iterations must see, use CommandCounterIncrement to
+ * update the visibility of those changes.
+ */
+ CommandCounterIncrement();
+ }
+
+ /* Close the relation and release the lock */
+ table_close(rel, NoLock);
+
+ /* Cleanup unused zstd dictionaries */
+ cleanup_unused_zstd_dictionaries_internal();
+
+ /* Pop the snapshot to clean up */
+ PopActiveSnapshot();
+
+ /* Commit the transaction */
+ CommitTransactionCommand();
+}
+
/*
* get_zstd_dict - Fetches the ZSTD dictionary from the catalog
*
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index db5da3ce826..87a5708788e 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/namespace.h"
#include "catalog/pg_database.h"
#include "catalog/pg_inherits.h"
+#include "catalog/pg_zstd_dictionaries.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
#include "commands/progress.h"
@@ -2312,6 +2313,9 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params,
vacuum_rel(toast_relid, NULL, &toast_vacuum_params, bstrategy);
}
+ /* Generate or cleanup unused ZSTD dictionaries. */
+ generate_or_cleanup_zstd_dictionaries_for_relation(relid);
+
/*
* Now release the session-level lock on the main table.
*/
diff --git a/src/include/catalog/pg_zstd_dictionaries.h b/src/include/catalog/pg_zstd_dictionaries.h
index cf847ee2801..a1e3948933c 100644
--- a/src/include/catalog/pg_zstd_dictionaries.h
+++ b/src/include/catalog/pg_zstd_dictionaries.h
@@ -44,5 +44,6 @@ DECLARE_UNIQUE_INDEX_PKEY(pg_zstd_dictionaries_dictid_index, 9949, ZstdDictidInd
MAKE_SYSCACHE(ZSTDDICTIDOID, pg_zstd_dictionaries_dictid_index, 128);
extern bytea *get_zstd_dict(Oid dictid);
+extern void generate_or_cleanup_zstd_dictionaries_for_relation(Oid relid);
#endif /* PG_ZSTD_DICTIONARIES_H */
--
2.47.1
[application/octet-stream] v11-0003-Zstd-dictionary-training-process.patch (41.5K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/3-v11-0003-Zstd-dictionary-training-process.patch)
download | inline diff:
From b63a58c68180872d882b5b3081ea40654e9062b3 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:45:42 +0000
Subject: [PATCH v11 3/7] Zstd dictionary training process
---
doc/src/sgml/ref/create_type.sgml | 21 +-
src/backend/catalog/heap.c | 10 +-
src/backend/catalog/pg_type.c | 11 +-
src/backend/catalog/pg_zstd_dictionaries.c | 490 ++++++++++++++++++++-
src/backend/commands/analyze.c | 7 +-
src/backend/commands/typecmds.c | 107 ++++-
src/backend/utils/cache/typcache.c | 2 +
src/include/catalog/pg_proc.dat | 35 ++
src/include/catalog/pg_type.dat | 4 +-
src/include/catalog/pg_type.h | 8 +-
src/include/parser/analyze.h | 5 +
src/include/utils/typcache.h | 1 +
src/test/regress/expected/oidjoins.out | 1 +
src/tools/pgindent/typedefs.list | 3 +
14 files changed, 680 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/ref/create_type.sgml b/doc/src/sgml/ref/create_type.sgml
index 994dfc65268..5f9d61db004 100644
--- a/doc/src/sgml/ref/create_type.sgml
+++ b/doc/src/sgml/ref/create_type.sgml
@@ -56,6 +56,7 @@ CREATE TYPE <replaceable class="parameter">name</replaceable> (
[ , ELEMENT = <replaceable class="parameter">element</replaceable> ]
[ , DELIMITER = <replaceable class="parameter">delimiter</replaceable> ]
[ , COLLATABLE = <replaceable class="parameter">collatable</replaceable> ]
+ [ , ZSTD_SAMPLING = <replaceable class="parameter">zstd_sampling_function</replaceable> ]
)
CREATE TYPE <replaceable class="parameter">name</replaceable>
@@ -211,7 +212,8 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
<replaceable class="parameter">type_modifier_input_function</replaceable>,
<replaceable class="parameter">type_modifier_output_function</replaceable>,
<replaceable class="parameter">analyze_function</replaceable>, and
- <replaceable class="parameter">subscript_function</replaceable>
+ <replaceable class="parameter">subscript_function</replaceable>, and
+ <replaceable class="parameter">zstd_sampling_function</replaceable>
are optional. Generally these functions have to be coded in C
or another low-level language.
</para>
@@ -491,6 +493,15 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
make use of the collation information; this does not happen
automatically merely by marking the type collatable.
</para>
+
+ <para>
+ The optional <replaceable class="parameter">zstd_sampling_function</replaceable>
+ performs type-specific sampling for a column of the corresponding data type.
+ By default, for <type>jsonb</type> data type, the function <literal>std_zstd_sampling_for_jsonb</literal> is defined. It attempts to gather samples for a jsonb
+ and returns a sample buffer for zstd dictionary training. The training function must be declared to accept two arguments of type <type>internal</type> and return an <type>internal</type> result.
+ The detailed information for zstd training function is provided in <filename>src/backend/catalog/pg_zstd_dictionaries.c</filename>.
+ </para>
+
</refsect2>
<refsect2 id="sql-createtype-array" xreflabel="Array Types">
@@ -846,6 +857,14 @@ CREATE TYPE <replaceable class="parameter">name</replaceable>
</para>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><replaceable class="parameter">zstd_sampling</replaceable></term>
+ <listitem>
+ <para>
+ Specifies the name of a function that performs sampling on specific type.
+ </para>
+ </listitem>
+ </varlistentry>
</variablelist>
</refsect1>
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index fbaed5359ad..9d3a8536e23 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -1071,7 +1071,10 @@ AddNewRelationType(const char *typeName,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ F_COMPOSITE_TYPZSTDSAMPLING /* generate dictionary
+ * procedure - default */
+ );
}
/* --------------------------------
@@ -1394,7 +1397,10 @@ heap_create_with_catalog(const char *relname,
-1, /* typmod */
0, /* array dimensions for typBaseType */
false, /* Type NOT NULL */
- InvalidOid); /* rowtypes never have a collation */
+ InvalidOid, /* rowtypes never have a collation */
+ F_ARRAY_TYPZSTDSAMPLING /* generate dictionary procedure -
+ * default */
+ );
pfree(relarrayname);
}
diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c
index b36f81afb9d..8ff9b4d327f 100644
--- a/src/backend/catalog/pg_type.c
+++ b/src/backend/catalog/pg_type.c
@@ -120,6 +120,7 @@ TypeShellMake(const char *typeName, Oid typeNamespace, Oid ownerId)
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(-1);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(0);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(InvalidOid);
+ values[Anum_pg_type_typzstdsampling - 1] = ObjectIdGetDatum(InvalidOid);
nulls[Anum_pg_type_typdefaultbin - 1] = true;
nulls[Anum_pg_type_typdefault - 1] = true;
nulls[Anum_pg_type_typacl - 1] = true;
@@ -223,7 +224,8 @@ TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims, /* Array dimensions for baseType */
bool typeNotNull,
- Oid typeCollation)
+ Oid typeCollation,
+ Oid zstdSamplingProcedure)
{
Relation pg_type_desc;
Oid typeObjectId;
@@ -378,6 +380,7 @@ TypeCreate(Oid newTypeOid,
values[Anum_pg_type_typtypmod - 1] = Int32GetDatum(typeMod);
values[Anum_pg_type_typndims - 1] = Int32GetDatum(typNDims);
values[Anum_pg_type_typcollation - 1] = ObjectIdGetDatum(typeCollation);
+ values[Anum_pg_type_typzstdsampling - 1] = ObjectIdGetDatum(zstdSamplingProcedure);
/*
* initialize the default binary value for this type. Check for nulls of
@@ -679,6 +682,12 @@ GenerateTypeDependencies(HeapTuple typeTuple,
add_exact_object_address(&referenced, addrs_normal);
}
+ if (OidIsValid(typeForm->typzstdsampling))
+ {
+ ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typzstdsampling);
+ add_exact_object_address(&referenced, addrs_normal);
+ }
+
if (OidIsValid(typeForm->typsubscript))
{
ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typsubscript);
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
index d5e965c34d0..58964a600a3 100644
--- a/src/backend/catalog/pg_zstd_dictionaries.c
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -14,10 +14,498 @@
#include "postgres.h"
#include "fmgr.h"
+#include "access/table.h"
+#include "catalog/dependency.h"
+#include "catalog/indexing.h"
+#include "catalog/pg_class_d.h"
#include "catalog/pg_zstd_dictionaries.h"
#include "catalog/pg_zstd_dictionaries_d.h"
+#include "catalog/pg_depend.h"
+#include "catalog/namespace.h"
+#include "catalog/pg_attribute.h"
+#include "utils/builtins.h"
+#include "utils/fmgroids.h"
+#include "utils/rel.h"
#include "utils/syscache.h"
-#include "varatt.h"
+#include "access/toast_compression.h"
+#include "utils/attoptcache.h"
+#include "parser/analyze.h"
+#include "nodes/makefuncs.h"
+#include "access/reloptions.h"
+#include "access/genam.h"
+#include "access/htup_details.h"
+#include "access/sdir.h"
+#include "utils/lsyscache.h"
+#include "utils/relcache.h"
+#include "utils/memutils.h"
+#include "utils/varlena.h"
+#include "nodes/pg_list.h"
+#include "utils/array.h"
+#include "utils/rangetypes.h"
+#include "utils/multirangetypes.h"
+#include "utils/snapmgr.h"
+#include "access/xact.h"
+
+#ifdef USE_ZSTD
+#include <zstd.h>
+#include <zdict.h>
+#endif
+
+#define TARGET_ROWS 30000
+
+typedef struct ZstdTrainingData ZstdTrainingData;
+
+struct ZstdTrainingData
+{
+ char *sample_buffer; /* Pointer to the raw sample buffer */
+ size_t *sample_sizes; /* Array of sample sizes */
+ size_t nitems; /* Number of sample sizes */
+ size_t total_size; /* Running total sample size */
+};
+
+static bool build_zstd_dictionary_internal(Oid relid, AttrNumber attno);
+static Oid GetNewDictId(Relation relation);
+static bool append_sample(ZstdTrainingData *dict, const char *sample, size_t sample_size);
+static bool sample_varlena_datum(Datum datum, ZstdTrainingData *dict);
+static int cleanup_unused_zstd_dictionaries_internal(void);
+
+/*
+ * build_zstd_dictionary_internal
+ * 1) Validate that the given (relid, attno) can have a Zstd compression enabled on heap relation
+ * 2) Call the type-specific sampling procedure
+ * 3) Train a dictionary via ZDICT_trainFromBuffer()
+ * 4) Insert dictionary into pg_zstd_dictionaries
+ * 5) Update pg_attribute.attoptions with new dictid
+ */
+pg_attribute_unused()
+static bool
+build_zstd_dictionary_internal(Oid relid, AttrNumber attno)
+{
+#ifndef USE_ZSTD
+ return false;
+#else
+ Relation rel;
+ Form_pg_attribute att;
+ AttributeOpts *attopt;
+ TypeCacheEntry *typentry;
+ ZstdTrainingData dict = {0};
+ HeapTuple sample_rows[TARGET_ROWS];
+ int num_sampled;
+ double totalrows = 0,
+ totaldeadrows = 0;
+ int i;
+ size_t dict_size;
+ void *dict_data;
+ Oid dictid;
+ bytea *dict_bytea;
+ Relation catalogRel,
+ attRel;
+ HeapTuple tup,
+ atttup,
+ newtuple;
+ Datum values[Natts_pg_zstd_dictionaries];
+ bool nulls[Natts_pg_zstd_dictionaries];
+ Datum attoptionsDatum,
+ newOptions;
+ bool isnull;
+ Datum repl_val[Natts_pg_attribute];
+ bool repl_null[Natts_pg_attribute];
+ bool repl_repl[Natts_pg_attribute];
+ DefElem *def;
+ ObjectAddress dictObj,
+ relObj;
+
+ /* Open relation, verify regular table */
+ rel = table_open(relid, AccessShareLock);
+ if (rel->rd_rel->relkind != RELKIND_RELATION)
+ goto fail;
+
+ att = TupleDescAttr(RelationGetDescr(rel), attno - 1);
+ if (att->attcompression != TOAST_ZSTD_COMPRESSION)
+ goto fail;
+
+ /* Check attoptions for user-requested dictionary size, etc. */
+ attopt = get_attribute_options(relid, attno);
+ if (attopt && attopt->zstd_dict_size == 0)
+ goto fail;
+
+ /*
+ * 2) Look up the type's custom dictionary builder function We'll call it
+ * to get sample training data.
+ */
+ typentry = lookup_type_cache(att->atttypid, 0);
+ if (typentry->typlen != -1 || !OidIsValid(typentry->typzstdsampling))
+ goto fail;
+
+ num_sampled = acquire_sample_rows(rel, 0, sample_rows, TARGET_ROWS, &totalrows, &totaldeadrows);
+ if (num_sampled == 0)
+ goto fail;
+
+ for (i = 0; i < num_sampled; i++)
+ {
+ Datum value;
+
+ value = heap_getattr(sample_rows[i], attno, RelationGetDescr(rel), &isnull);
+
+ if (!isnull && !DatumGetBool(OidFunctionCall2(typentry->typzstdsampling, value, PointerGetDatum(&dict))))
+ break;
+ }
+
+ if (dict.nitems == 0)
+ goto fail;
+
+ /* ZSTD Dictionary training */
+ dict_size = attopt ? attopt->zstd_dict_size : DEFAULT_ZSTD_DICT_SIZE;
+ dict_data = palloc(dict_size);
+
+ dict_size = ZDICT_trainFromBuffer(dict_data, dict_size, dict.sample_buffer, dict.sample_sizes, dict.nitems);
+ if (ZDICT_isError(dict_size))
+ {
+ elog(LOG, "Zstd dictionary training failed: %s", ZDICT_getErrorName(dict_size));
+ goto cleanup_dict;
+ }
+
+ /* Insert dictionary into catalog */
+ dict_bytea = palloc(VARHDRSZ + dict_size);
+ SET_VARSIZE(dict_bytea, VARHDRSZ + dict_size);
+ memcpy(VARDATA(dict_bytea), dict_data, dict_size);
+
+ catalogRel = table_open(ZstdDictionariesRelationId, ShareRowExclusiveLock);
+ dictid = GetNewDictId(catalogRel);
+
+ memset(values, 0, sizeof(values));
+ memset(nulls, 0, sizeof(nulls));
+ values[Anum_pg_zstd_dictionaries_dictid - 1] = ObjectIdGetDatum(dictid);
+ values[Anum_pg_zstd_dictionaries_dict - 1] = PointerGetDatum(dict_bytea);
+
+ tup = heap_form_tuple(RelationGetDescr(catalogRel), values, nulls);
+ CatalogTupleInsert(catalogRel, tup);
+
+ heap_freetuple(tup);
+ pfree(dict_bytea);
+ table_close(catalogRel, NoLock);
+
+ /*
+ * Update pg_attribute.attoptions with "dictid" => dictid so the column
+ * knows which dictionary to use at compression time.
+ */
+ attRel = table_open(AttributeRelationId, RowExclusiveLock);
+ atttup = SearchSysCacheAttNum(relid, attno);
+ if (!HeapTupleIsValid(atttup))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_COLUMN),
+ errmsg("column number %d of relation \"%u\" does not exist",
+ attno, relid)));
+
+ /* Build new attoptions with dictid=... */
+ def = makeDefElem("dictid",
+ (Node *) makeString(psprintf("%u", dictid)),
+ -1);
+
+ attoptionsDatum = SysCacheGetAttr(ATTNUM, atttup,
+ Anum_pg_attribute_attoptions,
+ &isnull);
+ newOptions = transformRelOptions(isnull ? (Datum) 0 : attoptionsDatum,
+ list_make1(def),
+ NULL, NULL,
+ false, false);
+ /* Validate them (throws error if invalid) */
+ (void) attribute_reloptions(newOptions, true);
+
+ memset(repl_null, false, sizeof(repl_null));
+ memset(repl_repl, false, sizeof(repl_repl));
+
+ repl_val[Anum_pg_attribute_attoptions - 1] = newOptions;
+ repl_repl[Anum_pg_attribute_attoptions - 1] = true;
+
+ newtuple = heap_modify_tuple(atttup, RelationGetDescr(attRel), repl_val, repl_null, repl_repl);
+ CatalogTupleUpdate(attRel, &newtuple->t_self, newtuple);
+
+ heap_freetuple(newtuple);
+ ReleaseSysCache(atttup);
+ table_close(attRel, NoLock);
+
+ /* Record dependency, relation is depended on this dictionary */
+ ObjectAddressSet(dictObj, ZstdDictionariesRelationId, dictid);
+ ObjectAddressSet(relObj, RelationRelationId, relid);
+ recordDependencyOn(&relObj, &dictObj, DEPENDENCY_NORMAL);
+
+ pfree(dict_data);
+ table_close(rel, NoLock);
+ return true;
+
+cleanup_dict:
+ pfree(dict_data);
+fail:
+ table_close(rel, NoLock);
+
+ return false;
+#endif
+}
+
+/*
+ * Acquire a new unique DictId for a relation.
+ *
+ * Assumes the relation is already locked with ShareRowExclusiveLock,
+ * ensuring that concurrent transactions cannot generate duplicate DictIds.
+ */
+pg_attribute_unused()
+static Oid
+GetNewDictId(Relation dictRel)
+{
+ Relation idxRel;
+ Oid maxDictId = InvalidOid;
+ Oid newDictId;
+ SysScanDesc scan;
+ HeapTuple tuple;
+
+ /*
+ * Open the index to read existing DictId values.
+ */
+ idxRel = index_open(ZstdDictidIndexId, AccessShareLock);
+
+ /*
+ * Retrieve the maximum existing DictId by scanning in reverse order. This
+ * relies on the index being sorted ascending on dictid, so scanning
+ * backward finds the largest value first.
+ */
+ scan = systable_beginscan_ordered(dictRel,
+ idxRel,
+ SnapshotSelf,
+ 0, NULL);
+
+ tuple = systable_getnext_ordered(scan, BackwardScanDirection);
+ if (HeapTupleIsValid(tuple))
+ {
+ Datum value;
+ bool isNull;
+
+ value = heap_getattr(tuple,
+ Anum_pg_zstd_dictionaries_dictid,
+ RelationGetDescr(dictRel),
+ &isNull);
+ if (!isNull)
+ maxDictId = DatumGetObjectId(value);
+ }
+ systable_endscan_ordered(scan);
+ index_close(idxRel, AccessShareLock);
+
+ /* Propose new DictId one higher than the max found. */
+ newDictId = maxDictId + 1;
+ Assert(newDictId != InvalidDictId);
+
+ if (newDictId <= InvalidDictId || newDictId > UINT32_MAX)
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("dictid is not in expected range")));
+
+ return newDictId;
+}
+
+/*
+ * append_sample
+ *
+ * Given a sample (raw bytes) and its size, append it to the training data.
+ * This function re-allocates (or allocates) the contiguous sample_buffer and
+ * the sample_sizes array. It returns true if the new total allocation does not
+ * exceed MaxAllocSize, false otherwise.
+ */
+static bool
+append_sample(ZstdTrainingData *dict, const char *sample, size_t sample_size)
+{
+ if ((dict->total_size + sample_size) > MaxAllocSize)
+ return false;
+
+ if (dict->sample_buffer == NULL)
+ dict->sample_buffer = palloc(sample_size);
+ else
+ dict->sample_buffer = repalloc(dict->sample_buffer, dict->total_size + sample_size);
+
+ memcpy(dict->sample_buffer + dict->total_size, sample, sample_size);
+ dict->total_size += sample_size;
+
+ if (dict->sample_sizes == NULL)
+ dict->sample_sizes = palloc(sizeof(size_t));
+ else
+ dict->sample_sizes = repalloc(dict->sample_sizes, (dict->nitems + 1) * sizeof(size_t));
+
+ dict->sample_sizes[dict->nitems++] = sample_size;
+
+ return true;
+}
+
+/* Common helper for jsonb and text */
+static bool
+sample_varlena_datum(Datum datum, ZstdTrainingData *dict)
+{
+ struct varlena *attr = (struct varlena *) PG_DETOAST_DATUM(datum);
+
+ return append_sample(dict, VARDATA_ANY(attr), VARSIZE_ANY_EXHDR(attr));
+}
+
+/*
+ * std_zstd_sampling_for_jsonb
+ *
+ * Processes a single jsonb sample.
+ * It detoasts the datum, obtains the raw sample (excluding the header),
+ * and appends it into the provided ZstdTrainingData structure.
+ *
+ * Returns true if the sample was successfully appended, false otherwise.
+ */
+Datum
+std_zstd_sampling_for_jsonb(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_BOOL(sample_varlena_datum(PG_GETARG_DATUM(0), (ZstdTrainingData *) PG_GETARG_POINTER(1)));
+}
+
+/*
+ * std_zstd_sampling_for_text
+ *
+ * Processes a single text sample.
+ * It detoasts the datum, obtains the raw sample (excluding the header),
+ * and appends it into the provided ZstdTrainingData structure.
+ *
+ * Returns true if the sample was successfully appended, false otherwise.
+ */
+Datum
+std_zstd_sampling_for_text(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_BOOL(sample_varlena_datum(PG_GETARG_DATUM(0), (ZstdTrainingData *) PG_GETARG_POINTER(1)));
+}
+
+/*
+ * array_typzstdsampling -- typzstdsampling function for array columns
+ */
+Datum
+array_typzstdsampling(PG_FUNCTION_ARGS)
+{
+ ArrayType *array = DatumGetArrayTypeP(PG_GETARG_DATUM(0));
+ ZstdTrainingData *dict = (ZstdTrainingData *) PG_GETARG_POINTER(1);
+ Datum *elements;
+ bool *nulls;
+ int nelems;
+ TypeCacheEntry *elemCache = lookup_type_cache(ARR_ELEMTYPE(array), 0);
+
+ if (!OidIsValid(elemCache->typzstdsampling))
+ PG_RETURN_BOOL(false);
+
+ deconstruct_array(array, ARR_ELEMTYPE(array), elemCache->typlen, elemCache->typbyval, elemCache->typalign, &elements, &nulls, &nelems);
+
+ for (int j = 0; j < nelems; j++)
+ if (!nulls[j] && !DatumGetBool(OidFunctionCall2(elemCache->typzstdsampling, elements[j], PointerGetDatum(dict))))
+ break;
+
+ pfree(elements);
+ pfree(nulls);
+ PG_RETURN_BOOL(true);
+}
+
+Datum
+range_typzstdsampling(PG_FUNCTION_ARGS)
+{
+ RangeType *range = DatumGetRangeTypeP(PG_GETARG_DATUM(0));
+ ZstdTrainingData *dict = (ZstdTrainingData *) PG_GETARG_POINTER(1);
+ RangeBound lower,
+ upper;
+ bool empty;
+
+ /* Get information about range type; note column might be a domain */
+ TypeCacheEntry *typcache = range_get_typcache(fcinfo, getBaseType(range->rangetypid));
+
+ /* If the type does not supply a builder, skip */
+ if (!OidIsValid(typcache->rngelemtype->typzstdsampling))
+ PG_RETURN_BOOL(false);
+
+ range_deserialize(typcache, range, &lower, &upper, &empty);
+ if (empty)
+ PG_RETURN_BOOL(false);
+
+ OidFunctionCall2(typcache->rngelemtype->typzstdsampling, lower.val, PointerGetDatum(dict));
+
+ OidFunctionCall2(typcache->rngelemtype->typzstdsampling, upper.val, PointerGetDatum(dict));
+
+ PG_RETURN_BOOL(true);
+}
+
+Datum
+multirange_typzstdsampling(PG_FUNCTION_ARGS)
+{
+ MultirangeType *mrange = DatumGetMultirangeTypeP(PG_GETARG_DATUM(0));
+ ZstdTrainingData *dict = (ZstdTrainingData *) PG_GETARG_POINTER(1);
+ int32 rangeCount;
+ RangeType **ranges;
+
+ /* Get information about multirange type; note column might be a domain */
+ TypeCacheEntry *typcache = multirange_get_typcache(fcinfo, getBaseType(mrange->multirangetypid));
+
+ /* If the type does not supply a builder, skip */
+ if (!OidIsValid(typcache->rngtype->typzstdsampling))
+ PG_RETURN_BOOL(false);
+
+ /* Deserialize the multirange into an array of RangeType pointers */
+ multirange_deserialize(typcache->rngtype, mrange, &rangeCount, &ranges);
+
+ for (int j = 0; j < rangeCount; j++)
+ if (!DatumGetBool(OidFunctionCall2(typcache->rngtype->typzstdsampling, RangeTypePGetDatum(ranges[j]), PointerGetDatum(dict))))
+ break;
+
+ PG_RETURN_BOOL(true);
+}
+
+Datum
+composite_typzstdsampling(PG_FUNCTION_ARGS)
+{
+ HeapTupleHeader th = DatumGetHeapTupleHeader(PG_GETARG_DATUM(0));
+ ZstdTrainingData *dict = (ZstdTrainingData *) PG_GETARG_POINTER(1);
+
+ TupleDesc tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(th), HeapTupleHeaderGetTypMod(th));
+ HeapTupleData tuple = {.t_data = th,.t_len = HeapTupleHeaderGetDatumLength(th),.t_tableOid = InvalidOid};
+
+ ItemPointerSetInvalid(&tuple.t_self);
+
+ for (int i = 0; i < tupdesc->natts; i++)
+ {
+ Form_pg_attribute attr = TupleDescAttr(tupdesc, i);
+ bool isnull;
+ Datum fieldDatum;
+
+ if (attr->attisdropped || attr->atthasmissing)
+ continue;
+
+ fieldDatum = heap_getattr(&tuple, i + 1, tupdesc, &isnull);
+
+ if (!isnull)
+ {
+ /* Look up the type cache entry for the attribute's type */
+ TypeCacheEntry *typcache = lookup_type_cache(attr->atttypid, 0);
+
+ if (OidIsValid(typcache->typzstdsampling) && !DatumGetBool(OidFunctionCall2(typcache->typzstdsampling, fieldDatum, PointerGetDatum(dict))))
+ break;
+ }
+ }
+
+ ReleaseTupleDesc(tupdesc);
+ PG_RETURN_BOOL(true);
+}
+
+Datum
+build_zstd_dict_for_attribute(PG_FUNCTION_ARGS)
+{
+#ifndef USE_ZSTD
+ PG_RETURN_BOOL(false);
+#else
+ text *tablename = PG_GETARG_TEXT_PP(0);
+ AttrNumber attno = PG_GETARG_INT32(1);
+
+ /* Look up table name. */
+ RangeVar *tablerel = makeRangeVarFromNameList(textToQualifiedNameList(tablename));
+ Oid tableoid = RangeVarGetRelid(tablerel, NoLock, false);
+
+ bool ret = build_zstd_dictionary_internal(tableoid, attno);
+
+ PG_RETURN_BOOL(ret);
+#endif
+}
/*
* get_zstd_dict - Fetches the ZSTD dictionary from the catalog
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 4fffb76e557..abb92a3a4fe 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -55,7 +55,7 @@
#include "utils/sortsupport.h"
#include "utils/syscache.h"
#include "utils/timestamp.h"
-
+#include "parser/analyze.h"
/* Per-index data for ANALYZE */
typedef struct AnlIndexData
@@ -85,9 +85,6 @@ static void compute_index_stats(Relation onerel, double totalrows,
MemoryContext col_context);
static VacAttrStats *examine_attribute(Relation onerel, int attnum,
Node *index_expr);
-static int acquire_sample_rows(Relation onerel, int elevel,
- HeapTuple *rows, int targrows,
- double *totalrows, double *totaldeadrows);
static int compare_rows(const void *a, const void *b, void *arg);
static int acquire_inherited_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
@@ -1195,7 +1192,7 @@ block_sampling_read_stream_next(ReadStream *stream,
* block. The previous sampling method put too much credence in the row
* density near the start of the table.
*/
-static int
+int
acquire_sample_rows(Relation onerel, int elevel,
HeapTuple *rows, int targrows,
double *totalrows, double *totaldeadrows)
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index 45ae7472ab5..8014ee507a7 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -95,6 +95,7 @@ typedef struct
bool updateTypmodout;
bool updateAnalyze;
bool updateSubscript;
+ bool updateZstdSampling;
/* New values for relevant attributes */
char storage;
Oid receiveOid;
@@ -103,6 +104,7 @@ typedef struct
Oid typmodoutOid;
Oid analyzeOid;
Oid subscriptOid;
+ Oid zstdSamplingOid;
} AlterTypeRecurseParams;
/* Potentially set by pg_upgrade_support functions */
@@ -122,6 +124,7 @@ static Oid findTypeSendFunction(List *procname, Oid typeOid);
static Oid findTypeTypmodinFunction(List *procname);
static Oid findTypeTypmodoutFunction(List *procname);
static Oid findTypeAnalyzeFunction(List *procname, Oid typeOid);
+static Oid findTypeZstdSamplingFunction(List *procname, Oid typeOid);
static Oid findTypeSubscriptingFunction(List *procname, Oid typeOid);
static Oid findRangeSubOpclass(List *opcname, Oid subtype);
static Oid findRangeCanonicalFunction(List *procname, Oid typeOid);
@@ -162,6 +165,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
List *typmodoutName = NIL;
List *analyzeName = NIL;
List *subscriptName = NIL;
+ List *zstdSamplingName = NIL;
char category = TYPCATEGORY_USER;
bool preferred = false;
char delimiter = DEFAULT_TYPDELIM;
@@ -190,6 +194,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
DefElem *alignmentEl = NULL;
DefElem *storageEl = NULL;
DefElem *collatableEl = NULL;
+ DefElem *zstdSamplingEl = NULL;
Oid inputOid;
Oid outputOid;
Oid receiveOid = InvalidOid;
@@ -198,6 +203,7 @@ DefineType(ParseState *pstate, List *names, List *parameters)
Oid typmodoutOid = InvalidOid;
Oid analyzeOid = InvalidOid;
Oid subscriptOid = InvalidOid;
+ Oid zstdSamplingOid = InvalidOid;
char *array_type;
Oid array_oid;
Oid typoid;
@@ -323,6 +329,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
defelp = &storageEl;
else if (strcmp(defel->defname, "collatable") == 0)
defelp = &collatableEl;
+ else if (strcmp(defel->defname, "zstd_sampling") == 0)
+ defelp = &zstdSamplingEl;
else
{
/* WARNING, not ERROR, for historical backwards-compatibility */
@@ -455,6 +463,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
}
if (collatableEl)
collation = defGetBoolean(collatableEl) ? DEFAULT_COLLATION_OID : InvalidOid;
+ if (zstdSamplingEl)
+ zstdSamplingName = defGetQualifiedName(zstdSamplingEl);
/*
* make sure we have our required definitions
@@ -516,6 +526,15 @@ DefineType(ParseState *pstate, List *names, List *parameters)
errmsg("element type cannot be specified without a subscripting function")));
}
+ if (zstdSamplingName)
+ {
+ if (internalLength != -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type zstd_sampling function must be specified only if data type is variable length.")));
+ zstdSamplingOid = findTypeZstdSamplingFunction(zstdSamplingName, typoid);
+ }
+
/*
* Check permissions on functions. We choose to require the creator/owner
* of a type to also own the underlying functions. Since creating a type
@@ -550,6 +569,9 @@ DefineType(ParseState *pstate, List *names, List *parameters)
if (analyzeOid && !object_ownercheck(ProcedureRelationId, analyzeOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(analyzeName));
+ if (zstdSamplingOid && !object_ownercheck(ProcedureRelationId, zstdSamplingOid, GetUserId()))
+ aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
+ NameListToString(zstdSamplingName));
if (subscriptOid && !object_ownercheck(ProcedureRelationId, subscriptOid, GetUserId()))
aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION,
NameListToString(subscriptName));
@@ -601,7 +623,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array Dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ zstdSamplingOid); /* zstd_sampling procedure */
Assert(typoid == address.objectId);
/*
@@ -643,7 +666,8 @@ DefineType(ParseState *pstate, List *names, List *parameters)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- collation); /* type's collation */
+ collation, /* type's collation */
+ F_ARRAY_TYPZSTDSAMPLING); /* zstd_sampling procedure */
pfree(array_type);
@@ -706,6 +730,7 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
Oid receiveProcedure;
Oid sendProcedure;
Oid analyzeProcedure;
+ Oid zstdSamplingOid;
bool byValue;
char category;
char delimiter;
@@ -842,6 +867,9 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
/* Analysis function */
analyzeProcedure = baseType->typanalyze;
+ /* Generate dictionary function */
+ zstdSamplingOid = baseType->typzstdsampling;
+
/*
* Domains don't need a subscript function, since they are not
* subscriptable on their own. If the base type is subscriptable, the
@@ -1078,7 +1106,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
basetypeMod, /* typeMod value */
typNDims, /* Array dimensions for base type */
typNotNull, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ zstdSamplingOid); /* zstd_sampling procedure */
/*
* Create the array type that goes with it.
@@ -1119,7 +1148,8 @@ DefineDomain(ParseState *pstate, CreateDomainStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- domaincoll); /* type's collation */
+ domaincoll, /* type's collation */
+ F_ARRAY_TYPZSTDSAMPLING); /* zstd_sampling procedure */
pfree(domainArrayName);
@@ -1241,7 +1271,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
/* Enter the enum's values into pg_enum */
EnumValuesCreate(enumTypeAddr.objectId, stmt->vals);
@@ -1282,7 +1313,8 @@ DefineEnum(CreateEnumStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation */
+ InvalidOid, /* type's collation */
+ InvalidOid); /* generate dictionary procedure - default */
pfree(enumArrayName);
@@ -1583,7 +1615,9 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ F_RANGE_TYPZSTDSAMPLING); /* generate dictionary
+ * procedure - default */
Assert(typoid == InvalidOid || typoid == address.objectId);
typoid = address.objectId;
@@ -1646,11 +1680,13 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
NULL, /* no binary form available either */
false, /* never passed by value */
alignment, /* alignment */
- 'x', /* TOAST strategy (always extended) */
+ TYPSTORAGE_EXTENDED, /* TOAST strategy (always extended) */
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* type's collation (ranges never have one) */
+ InvalidOid, /* type's collation (ranges never have one) */
+ F_MULTIRANGE_TYPZSTDSAMPLING); /* generate dictionary
+ * procedure - default */
Assert(multirangeOid == mltrngaddress.objectId);
/* Create the entry in pg_range */
@@ -1693,7 +1729,9 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ F_ARRAY_TYPZSTDSAMPLING); /* generate dictionary procedure -
+ * default */
pfree(rangeArrayName);
@@ -1728,11 +1766,13 @@ DefineRange(ParseState *pstate, CreateRangeStmt *stmt)
NULL, /* binary default isn't sent either */
false, /* never passed by value */
alignment, /* alignment - same as range's */
- 'x', /* ARRAY is always toastable */
+ TYPSTORAGE_EXTENDED, /* ARRAY is always toastable */
-1, /* typMod (Domains only) */
0, /* Array dimensions of typbasetype */
false, /* Type NOT NULL */
- InvalidOid); /* typcollation */
+ InvalidOid, /* typcollation */
+ F_ARRAY_TYPZSTDSAMPLING); /* generate dictionary procedure -
+ * default */
/* And create the constructor functions for this range type */
makeRangeConstructors(typeName, typeNamespace, typoid, rangeSubtype);
@@ -2261,6 +2301,31 @@ findTypeAnalyzeFunction(List *procname, Oid typeOid)
return procOid;
}
+static Oid
+findTypeZstdSamplingFunction(List *procname, Oid typeOid)
+{
+ Oid argList[2];
+ Oid procOid;
+
+ argList[0] = INTERNALOID;
+ argList[1] = INTERNALOID;
+
+ procOid = LookupFuncName(procname, 2, argList, true);
+ if (!OidIsValid(procOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_FUNCTION),
+ errmsg("function %s does not exist",
+ func_signature_string(procname, 2, NIL, argList))));
+
+ if (get_func_rettype(procOid) != BOOLOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("type build zstd dictionary function %s must return type %s",
+ NameListToString(procname), "internal")));
+
+ return procOid;
+}
+
static Oid
findTypeSubscriptingFunction(List *procname, Oid typeOid)
{
@@ -4444,6 +4509,19 @@ AlterType(AlterTypeStmt *stmt)
/* Replacing a subscript function requires superuser. */
requireSuper = true;
}
+ else if (strcmp(defel->defname, "zstd_sampling") == 0)
+ {
+ if (defel->arg != NULL)
+ atparams.zstdSamplingOid =
+ findTypeZstdSamplingFunction(defGetQualifiedName(defel),
+ typeOid);
+ else
+ atparams.zstdSamplingOid = InvalidOid; /* NONE, remove function */
+
+ atparams.updateZstdSampling = true;
+ /* Replacing a canonical function requires superuser. */
+ requireSuper = true;
+ }
/*
* The rest of the options that CREATE accepts cannot be changed.
@@ -4606,6 +4684,11 @@ AlterTypeRecurse(Oid typeOid, bool isImplicitArray,
replaces[Anum_pg_type_typsubscript - 1] = true;
values[Anum_pg_type_typsubscript - 1] = ObjectIdGetDatum(atparams->subscriptOid);
}
+ if (atparams->updateZstdSampling)
+ {
+ replaces[Anum_pg_type_typzstdsampling - 1] = true;
+ values[Anum_pg_type_typzstdsampling - 1] = ObjectIdGetDatum(atparams->zstdSamplingOid);
+ }
newtup = heap_modify_tuple(tup, RelationGetDescr(catalog),
values, nulls, replaces);
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index ae65a1cce06..7cc2d81cf42 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -501,6 +501,7 @@ lookup_type_cache(Oid type_id, int flags)
typentry->typelem = typtup->typelem;
typentry->typarray = typtup->typarray;
typentry->typcollation = typtup->typcollation;
+ typentry->typzstdsampling = typtup->typzstdsampling;
typentry->flags |= TCFLAGS_HAVE_PG_TYPE_DATA;
/* If it's a domain, immediately thread it into the domain cache list */
@@ -547,6 +548,7 @@ lookup_type_cache(Oid type_id, int flags)
typentry->typelem = typtup->typelem;
typentry->typarray = typtup->typarray;
typentry->typcollation = typtup->typcollation;
+ typentry->typzstdsampling = typtup->typzstdsampling;
typentry->flags |= TCFLAGS_HAVE_PG_TYPE_DATA;
ReleaseSysCache(tp);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 62beb71da28..7d2286850dc 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -12566,4 +12566,39 @@
proargnames => '{pid,io_id,io_generation,state,operation,off,length,target,handle_data_len,raw_result,result,target_desc,f_sync,f_localmem,f_buffered}',
prosrc => 'pg_get_aios' },
+# ZSTD related functions
+{ oid => '9241', descr => 'array typzstdsampling.',
+ proname => 'array_typzstdsampling', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'array_typzstdsampling' },
+
+{ oid => '9242', descr => 'range typzstdsampling.',
+ proname => 'range_typzstdsampling', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'range_typzstdsampling' },
+
+{ oid => '9243', descr => 'multirange typzstdsampling.',
+ proname => 'multirange_typzstdsampling', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'multirange_typzstdsampling' },
+
+{ oid => '9244', descr => 'composite typzstdsampling.',
+ proname => 'composite_typzstdsampling', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'composite_typzstdsampling' },
+
+{ oid => '9245', descr => 'Build zstd dictionaries for a column.',
+ proname => 'build_zstd_dict_for_attribute', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'text int4', proparallel => 'u',
+ prosrc => 'build_zstd_dict_for_attribute' },
+
+{ oid => '9247', descr => 'ZSTD standard sampling for jsonb',
+ proname => 'std_zstd_sampling_for_jsonb', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'std_zstd_sampling_for_jsonb' },
+
+{ oid => '9248', descr => 'ZSTD standard sampling for text',
+ proname => 'std_zstd_sampling_for_text', provolatile => 'v', prorettype => 'bool',
+ proargtypes => 'internal internal',
+ prosrc => 'std_zstd_sampling_for_text' },
]
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index 6dca77e0a22..a151ba33f82 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -83,7 +83,7 @@
typname => 'text', typlen => '-1', typbyval => 'f', typcategory => 'S',
typispreferred => 't', typinput => 'textin', typoutput => 'textout',
typreceive => 'textrecv', typsend => 'textsend', typalign => 'i',
- typstorage => 'x', typcollation => 'default' },
+ typstorage => 'x', typcollation => 'default', typzstdsampling => 'std_zstd_sampling_for_text' },
{ oid => '26', array_type_oid => '1028',
descr => 'object identifier(oid), maximum 4 billion',
typname => 'oid', typlen => '4', typbyval => 't', typcategory => 'N',
@@ -446,7 +446,7 @@
typname => 'jsonb', typlen => '-1', typbyval => 'f', typcategory => 'U',
typsubscript => 'jsonb_subscript_handler', typinput => 'jsonb_in',
typoutput => 'jsonb_out', typreceive => 'jsonb_recv', typsend => 'jsonb_send',
- typalign => 'i', typstorage => 'x' },
+ typalign => 'i', typstorage => 'x', typzstdsampling => 'std_zstd_sampling_for_jsonb' },
{ oid => '4072', array_type_oid => '4073', descr => 'JSON path',
typname => 'jsonpath', typlen => '-1', typbyval => 'f', typcategory => 'U',
typinput => 'jsonpath_in', typoutput => 'jsonpath_out',
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index ff666711a54..6f53b79feda 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -227,6 +227,11 @@ CATALOG(pg_type,1247,TypeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(71,TypeRelati
*/
Oid typcollation BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_collation);
+ /*
+ * Custom zstd sampling procedure for the datatype.
+ */
+ regproc typzstdsampling BKI_DEFAULT(0) BKI_ARRAY_DEFAULT(array_typzstdsampling) BKI_LOOKUP_OPT(pg_proc);
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/*
@@ -380,7 +385,8 @@ extern ObjectAddress TypeCreate(Oid newTypeOid,
int32 typeMod,
int32 typNDims,
bool typeNotNull,
- Oid typeCollation);
+ Oid typeCollation,
+ Oid zstdSamplingProcedure);
extern void GenerateTypeDependencies(HeapTuple typeTuple,
Relation typeCatalog,
diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h
index f29ed03b476..69391e40d0c 100644
--- a/src/include/parser/analyze.h
+++ b/src/include/parser/analyze.h
@@ -17,6 +17,7 @@
#include "nodes/params.h"
#include "nodes/queryjumble.h"
#include "parser/parse_node.h"
+#include "access/htup.h"
/* Hook for plugins to get control at end of parse analysis */
typedef void (*post_parse_analyze_hook_type) (ParseState *pstate,
@@ -65,4 +66,8 @@ extern List *BuildOnConflictExcludedTargetlist(Relation targetrel,
extern SortGroupClause *makeSortGroupClauseForSetOp(Oid rescoltype, bool require_hash);
+extern int acquire_sample_rows(Relation onerel, int elevel,
+ HeapTuple *rows, int targrows,
+ double *totalrows, double *totaldeadrows);
+
#endif /* ANALYZE_H */
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index 1cb30f1818c..c5bf668e519 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -46,6 +46,7 @@ typedef struct TypeCacheEntry
Oid typelem;
Oid typarray;
Oid typcollation;
+ Oid typzstdsampling;
/*
* Information obtained from opfamily entries
diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out
index 215eb899be3..09c750d04fa 100644
--- a/src/test/regress/expected/oidjoins.out
+++ b/src/test/regress/expected/oidjoins.out
@@ -71,6 +71,7 @@ NOTICE: checking pg_type {typmodout} => pg_proc {oid}
NOTICE: checking pg_type {typanalyze} => pg_proc {oid}
NOTICE: checking pg_type {typbasetype} => pg_type {oid}
NOTICE: checking pg_type {typcollation} => pg_collation {oid}
+NOTICE: checking pg_type {typzstdsampling} => pg_proc {oid}
NOTICE: checking pg_attribute {attrelid} => pg_class {oid}
NOTICE: checking pg_attribute {atttypid} => pg_type {oid}
NOTICE: checking pg_attribute {attcollation} => pg_collation {oid}
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 86fb79b2076..06a36903a83 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2641,6 +2641,8 @@ STARTUPINFO
STRLEN
SV
SYNCHRONIZATION_BARRIER
+SampleCollector
+SampleEntry
SampleScan
SampleScanGetSampleSize_function
SampleScanState
@@ -3370,6 +3372,7 @@ ZSTD_cParameter
ZSTD_inBuffer
ZSTD_outBuffer
ZstdCompressorState
+ZstdTrainingData
_SPI_connection
_SPI_plan
__m128i
--
2.47.1
[application/octet-stream] v11-0007-Some-tests-related-to-zstd-dictionary-based-comp.patch (71.9K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/4-v11-0007-Some-tests-related-to-zstd-dictionary-based-comp.patch)
download | inline diff:
From b98458770e75dd413d1a720bbd8ebf7ca38378d7 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:53:39 +0000
Subject: [PATCH v11 7/7] Some tests related to zstd dictionary based
compression and decompression
---
.../zstd-dictionary-build-cleanup.out | 661 ++++++++++++++++
.../zstd-dictionary-build-cleanup_2.out | 709 ++++++++++++++++++
.../zstd-dictionary-tblcpy-cleanup.out | 199 +++++
.../zstd-dictionary-tblcpy-cleanup_2.out | 161 ++++
src/test/isolation/isolation_schedule | 2 +
.../specs/zstd-dictionary-build-cleanup.spec | 40 +
.../specs/zstd-dictionary-tblcpy-cleanup.spec | 46 ++
.../regress/expected/compression_zstd.out | 158 ++++
.../regress/expected/compression_zstd_1.out | 251 +++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/compression_zstd.sql | 106 +++
11 files changed, 2334 insertions(+), 1 deletion(-)
create mode 100644 src/test/isolation/expected/zstd-dictionary-build-cleanup.out
create mode 100644 src/test/isolation/expected/zstd-dictionary-build-cleanup_2.out
create mode 100644 src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup.out
create mode 100644 src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup_2.out
create mode 100644 src/test/isolation/specs/zstd-dictionary-build-cleanup.spec
create mode 100644 src/test/isolation/specs/zstd-dictionary-tblcpy-cleanup.spec
create mode 100644 src/test/regress/expected/compression_zstd.out
create mode 100644 src/test/regress/expected/compression_zstd_1.out
create mode 100644 src/test/regress/sql/compression_zstd.sql
diff --git a/src/test/isolation/expected/zstd-dictionary-build-cleanup.out b/src/test/isolation/expected/zstd-dictionary-build-cleanup.out
new file mode 100644
index 00000000000..f68c27e3713
--- /dev/null
+++ b/src/test/isolation/expected/zstd-dictionary-build-cleanup.out
@@ -0,0 +1,661 @@
+Parsed test spec with 4 sessions
+
+starting permutation: insert_more build run_cleanup run_droptbl
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: insert_more build run_droptbl run_cleanup
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_cleanup build run_droptbl
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: insert_more run_cleanup run_droptbl build
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_droptbl build run_cleanup
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_droptbl run_cleanup build
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build insert_more run_cleanup run_droptbl
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: build insert_more run_droptbl run_cleanup
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_cleanup insert_more run_droptbl
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: build run_cleanup run_droptbl insert_more
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: build run_droptbl insert_more run_cleanup
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_droptbl run_cleanup insert_more
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup insert_more build run_droptbl
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: run_cleanup insert_more run_droptbl build
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup build insert_more run_droptbl
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: run_cleanup build run_droptbl insert_more
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 1
+(1 row)
+
+
+starting permutation: run_cleanup run_droptbl insert_more build
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup run_droptbl build insert_more
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl insert_more build run_cleanup
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl insert_more run_cleanup build
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl build insert_more run_cleanup
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl build run_cleanup insert_more
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl run_cleanup insert_more build
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl run_cleanup build insert_more
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+ERROR: relation "messages" does not exist
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
diff --git a/src/test/isolation/expected/zstd-dictionary-build-cleanup_2.out b/src/test/isolation/expected/zstd-dictionary-build-cleanup_2.out
new file mode 100644
index 00000000000..b4d5705b969
--- /dev/null
+++ b/src/test/isolation/expected/zstd-dictionary-build-cleanup_2.out
@@ -0,0 +1,709 @@
+Parsed test spec with 4 sessions
+
+starting permutation: insert_more build run_cleanup run_droptbl
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more build run_droptbl run_cleanup
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_cleanup build run_droptbl
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_cleanup run_droptbl build
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_droptbl build run_cleanup
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_more run_droptbl run_cleanup build
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build insert_more run_cleanup run_droptbl
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build insert_more run_droptbl run_cleanup
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_cleanup insert_more run_droptbl
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_cleanup run_droptbl insert_more
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_droptbl insert_more run_cleanup
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: build run_droptbl run_cleanup insert_more
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup insert_more build run_droptbl
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup insert_more run_droptbl build
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup build insert_more run_droptbl
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup build run_droptbl insert_more
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup run_droptbl insert_more build
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_cleanup run_droptbl build insert_more
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl insert_more build run_cleanup
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl insert_more run_cleanup build
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl build insert_more run_cleanup
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl build run_cleanup insert_more
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl run_cleanup insert_more build
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: run_droptbl run_cleanup build insert_more
+step run_droptbl:
+ DROP TABLE IF EXISTS messages;
+
+step run_cleanup:
+ SELECT cleanup_unused_zstd_dictionaries();
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+step build:
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+step insert_more:
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+
+ERROR: relation "messages" does not exist
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
diff --git a/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup.out b/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup.out
new file mode 100644
index 00000000000..b0cef50c280
--- /dev/null
+++ b/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup.out
@@ -0,0 +1,199 @@
+Parsed test spec with 2 sessions
+
+starting permutation: build_dict_and_insert insert_into_othertbl build_dict_and_insert insert_into_othertbl
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+objid |refobjid
+-------------+--------
+messages_temp| 1
+(1 row)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid |refobjid
+-------------+--------
+messages_temp| 1
+(1 row)
+
+datum_leak: NOTICE: table "messages_temp_ctas" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas1" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas2" does not exist, skipping
+objid |refobjid
+-------------------+--------
+messages_temp | 1
+messages_temp_cpy | 1
+messages_temp_ctas | 1
+messages_temp_ctas1| 1
+messages_temp_ctas2| 1
+(5 rows)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid |refobjid
+-------------------+--------
+messages_temp | 1
+messages_temp_cpy | 1
+messages_temp_ctas | 1
+messages_temp_ctas1| 1
+messages_temp_ctas2| 1
+(5 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+objid |refobjid
+-------------------+--------
+messages_temp | 1
+messages_temp | 2
+messages_temp_cpy | 1
+messages_temp_ctas | 1
+messages_temp_ctas1| 1
+messages_temp_ctas2| 1
+(6 rows)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid |refobjid
+-------------------+--------
+messages_temp | 1
+messages_temp | 2
+messages_temp_cpy | 1
+messages_temp_ctas | 1
+messages_temp_ctas1| 1
+messages_temp_ctas2| 1
+(6 rows)
+
+objid |refobjid
+-------------------+--------
+messages_temp | 1
+messages_temp | 2
+messages_temp_cpy | 1
+messages_temp_cpy | 2
+messages_temp_ctas | 1
+messages_temp_ctas | 2
+messages_temp_ctas1| 1
+messages_temp_ctas1| 2
+messages_temp_ctas2| 1
+messages_temp_ctas2| 2
+(10 rows)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 2
+(1 row)
+
+
+starting permutation: insert_into_othertbl insert_into_othertbl build_dict_and_insert build_dict_and_insert
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+datum_leak: NOTICE: table "messages_temp_ctas" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas1" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas2" does not exist, skipping
+objid|refobjid
+-----+--------
+(0 rows)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+objid |refobjid
+-------------+--------
+messages_temp| 1
+(1 row)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid |refobjid
+-------------+--------
+messages_temp| 1
+(1 row)
+
+build_zstd_dict_for_attribute
+-----------------------------
+t
+(1 row)
+
+objid |refobjid
+-------------+--------
+messages_temp| 1
+messages_temp| 2
+(2 rows)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 2
+(1 row)
+
diff --git a/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup_2.out b/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup_2.out
new file mode 100644
index 00000000000..8229150eb64
--- /dev/null
+++ b/src/test/isolation/expected/zstd-dictionary-tblcpy-cleanup_2.out
@@ -0,0 +1,161 @@
+Parsed test spec with 2 sessions
+
+starting permutation: build_dict_and_insert insert_into_othertbl build_dict_and_insert insert_into_othertbl
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+datum_leak: NOTICE: table "messages_temp_ctas" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas1" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas2" does not exist, skipping
+objid|refobjid
+-----+--------
+(0 rows)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
+
+starting permutation: insert_into_othertbl insert_into_othertbl build_dict_and_insert build_dict_and_insert
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+datum_leak: NOTICE: table "messages_temp_ctas" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas1" does not exist, skipping
+datum_leak: NOTICE: table "messages_temp_ctas2" does not exist, skipping
+objid|refobjid
+-----+--------
+(0 rows)
+
+step insert_into_othertbl:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+step build_dict_and_insert:
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+build_zstd_dict_for_attribute
+-----------------------------
+f
+(1 row)
+
+objid|refobjid
+-----+--------
+(0 rows)
+
+cleanup_unused_zstd_dictionaries
+--------------------------------
+ 0
+(1 row)
+
diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule
index e3c669a29c7..fd446e8f357 100644
--- a/src/test/isolation/isolation_schedule
+++ b/src/test/isolation/isolation_schedule
@@ -116,3 +116,5 @@ test: serializable-parallel-2
test: serializable-parallel-3
test: matview-write-skew
test: lock-nowait
+test: zstd-dictionary-build-cleanup
+test: zstd-dictionary-tblcpy-cleanup
\ No newline at end of file
diff --git a/src/test/isolation/specs/zstd-dictionary-build-cleanup.spec b/src/test/isolation/specs/zstd-dictionary-build-cleanup.spec
new file mode 100644
index 00000000000..f526ba79d72
--- /dev/null
+++ b/src/test/isolation/specs/zstd-dictionary-build-cleanup.spec
@@ -0,0 +1,40 @@
+# Zstd compression race test for dictionary build and cleanup methods
+
+setup
+{
+ CREATE TABLE messages (content text);
+ DO $$
+ BEGIN
+ ALTER TABLE messages ALTER COLUMN content SET COMPRESSION ZSTD;
+ EXCEPTION WHEN others THEN
+ RAISE WARNING 'Ignoring zstd compression: this build does not support it.';
+ END
+ $$;
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 10);
+}
+
+teardown
+{
+ DROP TABLE IF EXISTS messages;
+ SELECT cleanup_unused_zstd_dictionaries();
+}
+
+session "insert"
+step "insert_more" {
+ INSERT INTO messages SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+}
+
+session "build_dict"
+step "build" {
+ SELECT build_zstd_dict_for_attribute('messages', 1);
+}
+
+session "cleanup"
+step "run_cleanup" {
+ SELECT cleanup_unused_zstd_dictionaries();
+}
+
+session "droptbls"
+step "run_droptbl" {
+ DROP TABLE IF EXISTS messages;
+}
diff --git a/src/test/isolation/specs/zstd-dictionary-tblcpy-cleanup.spec b/src/test/isolation/specs/zstd-dictionary-tblcpy-cleanup.spec
new file mode 100644
index 00000000000..57aabdf1cf7
--- /dev/null
+++ b/src/test/isolation/specs/zstd-dictionary-tblcpy-cleanup.spec
@@ -0,0 +1,46 @@
+# Zstd compression test for dictionary build, zstd table copy and cleanup method
+
+setup
+{
+ CREATE TABLE messages_temp (content text);
+ DO $$
+ BEGIN
+ ALTER TABLE messages_temp ALTER COLUMN content SET COMPRESSION ZSTD;
+ EXCEPTION WHEN others THEN
+ RAISE WARNING 'Ignoring zstd compression: this build does not support it.';
+ END
+ $$;
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 10);
+ CREATE TABLE messages_temp_cpy (like messages_temp INCLUDING ALL);
+}
+
+teardown
+{
+ DROP TABLE IF EXISTS messages_temp;
+ DROP TABLE IF EXISTS messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas1;
+ DROP TABLE IF EXISTS messages_temp_ctas2;
+ SELECT cleanup_unused_zstd_dictionaries();
+}
+
+session "build_dict_and_insert"
+step "build_dict_and_insert" {
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ SELECT build_zstd_dict_for_attribute('messages_temp', 1);
+ INSERT INTO messages_temp SELECT repeat('Random_', 500) FROM generate_series(1, 5);
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+}
+
+session "datum_leak"
+step "insert_into_othertbl" {
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+ INSERT INTO messages_temp_cpy SELECT * from messages_temp;
+ DROP TABLE IF EXISTS messages_temp_ctas; create table messages_temp_ctas as SELECT * from messages_temp_cpy;
+ DROP TABLE IF EXISTS messages_temp_ctas1; create table messages_temp_ctas1 as SELECT * from messages_temp_ctas;
+ DROP TABLE IF EXISTS messages_temp_ctas2; create table messages_temp_ctas2 as SELECT * from messages_temp_ctas1;
+ SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946 order by objid::regclass, refobjid;
+}
+
+permutation build_dict_and_insert insert_into_othertbl build_dict_and_insert insert_into_othertbl
+permutation insert_into_othertbl insert_into_othertbl build_dict_and_insert build_dict_and_insert
diff --git a/src/test/regress/expected/compression_zstd.out b/src/test/regress/expected/compression_zstd.out
new file mode 100644
index 00000000000..8fee9961abe
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd.out
@@ -0,0 +1,158 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+ERROR: compression method zstd not supported
+DETAIL: This functionality requires the server to be built with zstd support.
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 10...
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+-- Train dictionary for f1 column and insert more.
+SELECT build_zstd_dict_for_attribute('cmdata_zstd', 1);
+ build_zstd_dict_for_attribute
+-------------------------------
+ f
+(1 row)
+
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 10...
+ ^
+QUERY: INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004))
+CONTEXT: PL/pgSQL function inline_code_block line 4 at SQL statement
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------+----------
+(0 rows)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+(0 rows)
+
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 3: FROM cmdata_zstd
+ ^
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPR...
+ ^
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+ERROR: table "cmdata_zstd_2" does not exist
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: SELECT f1 FROM cmdata_zstd;
+ ^
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ERROR: relation "compressmv_zstd" does not exist
+LINE 2: FROM compressmv_zstd;
+ ^
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------+----------
+(0 rows)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+(0 rows)
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+ERROR: relation "cmdata_zstd" does not exist
+LINE 1: UPDATE cmdata_zstd
+ ^
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ERROR: relation "cmdata_zstd" does not exist
+LINE 2: FROM cmdata_zstd;
+ ^
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+ERROR: materialized view "compressmv_zstd" does not exist
+DROP TABLE cmdata_zstd;
+ERROR: table "cmdata_zstd" does not exist
+--cleanup dictionary
+select cleanup_unused_zstd_dictionaries();
+ cleanup_unused_zstd_dictionaries
+----------------------------------
+ 0
+(1 row)
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------+----------
+(0 rows)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+(0 rows)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out
new file mode 100644
index 00000000000..c1a7936e574
--- /dev/null
+++ b/src/test/regress/expected/compression_zstd_1.out
@@ -0,0 +1,251 @@
+\set HIDE_TOAST_COMPRESSION false
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+NOTICE: table "cmdata_zstd" does not exist, skipping
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+-- Train dictionary for f1 column and insert more.
+SELECT build_zstd_dict_for_attribute('cmdata_zstd', 1);
+ build_zstd_dict_for_attribute
+-------------------------------
+ t
+(1 row)
+
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------------+----------
+ cmdata_zstd | 1
+(1 row)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+ 1
+(1 row)
+
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+ Table "public.cmdata_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+ compression_method | row_count
+--------------------+-----------
+ zstd | 30
+(1 row)
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+ data_slice
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+ 01234567890123456789012345678901234567890123456789
+(30 rows)
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+NOTICE: table "cmdata_zstd_2" does not exist, skipping
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+ Table "public.cmdata_zstd_2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | zstd | |
+
+DROP TABLE cmdata_zstd_2;
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+NOTICE: materialized view "compressmv_zstd" does not exist, skipping
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+ Materialized view "public.compressmv_zstd"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | | |
+View definition:
+ SELECT f1
+ FROM cmdata_zstd;
+
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+ mv_compression
+----------------
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+ zstd
+(30 rows)
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------------+----------
+ cmdata_zstd | 1
+(1 row)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+ 1
+(1 row)
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+ preview
+---------
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+ UPDATED
+(30 rows)
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+--cleanup dictionary
+select cleanup_unused_zstd_dictionaries();
+ cleanup_unused_zstd_dictionaries
+----------------------------------
+ 1
+(1 row)
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+ objid | refobjid
+-------+----------
+(0 rows)
+
+select dictid from pg_zstd_dictionaries;
+ dictid
+--------
+(0 rows)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 0f38caa0d24..4df32357d01 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -119,7 +119,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# The stats test resets stats, so nothing else needing stats access can be in
# this group.
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression memoize stats predicate numa
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression compression_zstd memoize stats predicate numa
# event_trigger depends on create_am and cannot run concurrently with
# any test that runs DDL
diff --git a/src/test/regress/sql/compression_zstd.sql b/src/test/regress/sql/compression_zstd.sql
new file mode 100644
index 00000000000..894e72da0cb
--- /dev/null
+++ b/src/test/regress/sql/compression_zstd.sql
@@ -0,0 +1,106 @@
+\set HIDE_TOAST_COMPRESSION false
+
+-- Ensure stable results regardless of the installation's default.
+SET default_toast_compression = 'pglz';
+
+----------------------------------------------------------------
+-- 1. Create Test Table with Zstd Compression
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd CASCADE;
+CREATE TABLE cmdata_zstd (
+ f1 TEXT COMPRESSION zstd
+);
+
+----------------------------------------------------------------
+-- 2. Insert Data Rows
+----------------------------------------------------------------
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+
+-- Train dictionary for f1 column and insert more.
+SELECT build_zstd_dict_for_attribute('cmdata_zstd', 1);
+
+DO $$
+BEGIN
+ FOR i IN 1..15 LOOP
+ INSERT INTO cmdata_zstd (f1) VALUES (repeat('1234567890', 1004));
+ END LOOP;
+END $$;
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+
+select dictid from pg_zstd_dictionaries;
+
+----------------------------------------------------------------
+-- 3. Verify Table Structure and Compression Settings
+----------------------------------------------------------------
+-- Table Structure for cmdata_zstd
+\d+ cmdata_zstd;
+
+-- Compression Settings for f1 Column
+SELECT pg_column_compression(f1) AS compression_method,
+ count(*) AS row_count
+FROM cmdata_zstd
+GROUP BY pg_column_compression(f1);
+
+----------------------------------------------------------------
+-- 4. Decompression Tests
+----------------------------------------------------------------
+-- Decompression Slice Test (Extracting Substrings)
+SELECT SUBSTR(f1, 200, 50) AS data_slice
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 5. Test Table Creation with LIKE INCLUDING COMPRESSION
+----------------------------------------------------------------
+DROP TABLE IF EXISTS cmdata_zstd_2;
+CREATE TABLE cmdata_zstd_2 (LIKE cmdata_zstd INCLUDING COMPRESSION);
+-- Table Structure for cmdata_zstd_2
+\d+ cmdata_zstd_2;
+DROP TABLE cmdata_zstd_2;
+
+----------------------------------------------------------------
+-- 6. Materialized View Compression Test
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW IF EXISTS compressmv_zstd;
+CREATE MATERIALIZED VIEW compressmv_zstd AS
+ SELECT f1 FROM cmdata_zstd;
+-- Materialized View Structure for compressmv_zstd
+\d+ compressmv_zstd;
+-- Materialized View Compression Check
+SELECT pg_column_compression(f1) AS mv_compression
+FROM compressmv_zstd;
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+
+select dictid from pg_zstd_dictionaries;
+
+----------------------------------------------------------------
+-- 7. Additional Updates and Round-Trip Tests
+----------------------------------------------------------------
+-- Update some rows to check if the dictionary remains effective after modifications.
+UPDATE cmdata_zstd
+SET f1 = f1 || ' UPDATED';
+
+-- Verification of Updated Rows
+SELECT SUBSTR(f1, LENGTH(f1) - 7 + 1, 7) AS preview
+FROM cmdata_zstd;
+
+----------------------------------------------------------------
+-- 8. Clean Up
+----------------------------------------------------------------
+DROP MATERIALIZED VIEW compressmv_zstd;
+DROP TABLE cmdata_zstd;
+
+--cleanup dictionary
+select cleanup_unused_zstd_dictionaries();
+
+SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
+
+select dictid from pg_zstd_dictionaries;
+
+\set HIDE_TOAST_COMPRESSION true
--
2.47.1
[application/octet-stream] v11-0006-pg_dump-pg_upgrade-needed-changes-to-support-new.patch (13.5K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/5-v11-0006-pg_dump-pg_upgrade-needed-changes-to-support-new.patch)
download | inline diff:
From 60625c15c53b0e5e0090f98b2d692f03f9d617cd Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:53:02 +0000
Subject: [PATCH v11 6/7] pg_dump, pg_upgrade needed changes to support new
zstd catalog
---
src/bin/pg_dump/pg_dump.c | 210 ++++++++++++++++++++++++++++++++-----
src/bin/pg_upgrade/check.c | 30 ++++--
src/bin/pg_upgrade/info.c | 2 +-
3 files changed, 209 insertions(+), 33 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index c6e6d3b2b86..539e046cf22 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -53,6 +53,7 @@
#include "catalog/pg_publication_d.h"
#include "catalog/pg_subscription_d.h"
#include "catalog/pg_type_d.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
#include "common/connect.h"
#include "common/int.h"
#include "common/relpath.h"
@@ -3646,6 +3647,144 @@ dumpDatabase(Archive *fout)
destroyPQExpBuffer(loOutQry);
}
+ if (dopt->binary_upgrade && fout->remoteVersion >= 180000)
+ {
+ /**
+ * --- Process pg_zstd_dictionaries related operations ---
+ */
+ {
+ PGresult *zstd_res = NULL;
+ PQExpBuffer zstdFrozenQry = createPQExpBuffer();
+ PQExpBuffer zstdOutQry = createPQExpBuffer();
+ PQExpBuffer zstdHorizonQry = createPQExpBuffer();
+ int ii_relfrozenxid,
+ ii_relfilenode,
+ ii_oid,
+ ii_relminmxid;
+
+ /* Build query to fetch pg_class information */
+ appendPQExpBuffer(zstdFrozenQry,
+ "SELECT relfrozenxid, relminmxid, relfilenode, oid\n"
+ "FROM pg_catalog.pg_class\n"
+ "WHERE oid IN (%u, %u, %u, %u);\n",
+ ZstdDictionariesRelationId,
+ PgZstdDictionariesToastTable, /* toast table */
+ PgZstdDictionariesToastIndex, /* toast table index */
+ ZstdDictidIndexId); /* index */
+
+ zstd_res = ExecuteSqlQuery(fout, zstdFrozenQry->data, PGRES_TUPLES_OK);
+
+ ii_relfrozenxid = PQfnumber(zstd_res, "relfrozenxid");
+ ii_relminmxid = PQfnumber(zstd_res, "relminmxid");
+ ii_relfilenode = PQfnumber(zstd_res, "relfilenode");
+ ii_oid = PQfnumber(zstd_res, "oid");
+
+ appendPQExpBufferStr(zstdHorizonQry,
+ "\n-- For binary upgrade, set pg_zstd_dictionaries relfilenode, relfrozenxid, and relminmxid\n");
+ appendPQExpBufferStr(zstdOutQry,
+ "\n-- For binary upgrade, preserve pg_zstd_dictionaries and related relfilenodes\n");
+
+ /* Loop over each result row and build update statements */
+ for (int i = 0; i < PQntuples(zstd_res); ++i)
+ {
+ Oid oid = atooid(PQgetvalue(zstd_res, i, ii_oid));
+ Oid relfilenumber = atooid(PQgetvalue(zstd_res, i, ii_relfilenode));
+
+ appendPQExpBuffer(zstdHorizonQry,
+ "UPDATE pg_catalog.pg_class\n"
+ "SET relfrozenxid = '%u', relminmxid = '%u', relfilenode = %u\n"
+ "WHERE oid = %u;\n",
+ atooid(PQgetvalue(zstd_res, i, ii_relfrozenxid)),
+ atooid(PQgetvalue(zstd_res, i, ii_relminmxid)),
+ relfilenumber,
+ oid);
+
+ if (oid == ZstdDictionariesRelationId || oid == PgZstdDictionariesToastTable)
+ appendPQExpBuffer(zstdOutQry,
+ "SELECT pg_catalog.binary_upgrade_set_next_heap_relfilenode('%u'::pg_catalog.oid);\n",
+ relfilenumber);
+ else if (oid == PgZstdDictionariesToastIndex || oid == ZstdDictidIndexId)
+ appendPQExpBuffer(zstdOutQry,
+ "SELECT pg_catalog.binary_upgrade_set_next_index_relfilenode('%u'::pg_catalog.oid);\n",
+ relfilenumber);
+ }
+
+ appendPQExpBufferStr(zstdOutQry, zstdHorizonQry->data);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = "pg_zstd_dictionaries",
+ .description = "pg_zstd_dictionaries",
+ .section = SECTION_PRE_DATA,
+ .createStmt = zstdOutQry->data));
+
+ PQclear(zstd_res);
+ destroyPQExpBuffer(zstdFrozenQry);
+ destroyPQExpBuffer(zstdHorizonQry);
+ destroyPQExpBuffer(zstdOutQry);
+ }
+
+ /**
+ * --- Process pg_depend related operations ---
+ */
+ {
+ PGresult *pgdep_res = NULL;
+ PQExpBuffer pgdepQry = createPQExpBuffer();
+ PQExpBuffer pgdepBuf = createPQExpBuffer();
+ int i_classid,
+ i_objid,
+ i_objsubid,
+ i_refclassid,
+ i_refobjid,
+ i_refobjsubid,
+ i_deptype;
+
+ /*
+ * Build query to fetch all dependency rows where refclassid
+ * equals ZstdDictionariesRelationId
+ */
+ appendPQExpBuffer(pgdepQry,
+ "SELECT classid, objid, objsubid, refclassid, refobjid, refobjsubid, deptype\n"
+ "FROM pg_catalog.pg_depend\n"
+ "WHERE refclassid = %u;\n",
+ ZstdDictionariesRelationId);
+
+ pgdep_res = ExecuteSqlQuery(fout, pgdepQry->data, PGRES_TUPLES_OK);
+
+ i_classid = PQfnumber(pgdep_res, "classid");
+ i_objid = PQfnumber(pgdep_res, "objid");
+ i_objsubid = PQfnumber(pgdep_res, "objsubid");
+ i_refclassid = PQfnumber(pgdep_res, "refclassid");
+ i_refobjid = PQfnumber(pgdep_res, "refobjid");
+ i_refobjsubid = PQfnumber(pgdep_res, "refobjsubid");
+ i_deptype = PQfnumber(pgdep_res, "deptype");
+
+ /* Loop over dependency rows and generate INSERT statements */
+ for (int i = 0; i < PQntuples(pgdep_res); i++)
+ {
+ appendPQExpBuffer(pgdepBuf,
+ "INSERT INTO pg_catalog.pg_depend (classid, objid, objsubid, refclassid, refobjid, refobjsubid, deptype) "
+ "VALUES (%s, %s, %s, %s, %s, %s, '%s');\n",
+ PQgetvalue(pgdep_res, i, i_classid),
+ PQgetvalue(pgdep_res, i, i_objid),
+ PQgetvalue(pgdep_res, i, i_objsubid),
+ PQgetvalue(pgdep_res, i, i_refclassid),
+ PQgetvalue(pgdep_res, i, i_refobjid),
+ PQgetvalue(pgdep_res, i, i_refobjsubid),
+ PQgetvalue(pgdep_res, i, i_deptype));
+ }
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = "pg_depend_refclassid_9946",
+ .description = "pg_depend rows for refclassid 9946",
+ .section = SECTION_DATA,
+ .createStmt = pgdepBuf->data));
+
+ PQclear(pgdep_res);
+ destroyPQExpBuffer(pgdepQry);
+ destroyPQExpBuffer(pgdepBuf);
+ }
+ }
+
PQclear(res);
free(qdatname);
@@ -9061,29 +9200,32 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
* collation is different from their type's default, we use a CASE here to
* suppress uninteresting attcollations cheaply.
*/
- appendPQExpBufferStr(q,
- "SELECT\n"
- "a.attrelid,\n"
- "a.attnum,\n"
- "a.attname,\n"
- "a.attstattarget,\n"
- "a.attstorage,\n"
- "t.typstorage,\n"
- "a.atthasdef,\n"
- "a.attisdropped,\n"
- "a.attlen,\n"
- "a.attalign,\n"
- "a.attislocal,\n"
- "pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n"
- "array_to_string(a.attoptions, ', ') AS attoptions,\n"
- "CASE WHEN a.attcollation <> t.typcollation "
- "THEN a.attcollation ELSE 0 END AS attcollation,\n"
- "pg_catalog.array_to_string(ARRAY("
- "SELECT pg_catalog.quote_ident(option_name) || "
- "' ' || pg_catalog.quote_literal(option_value) "
- "FROM pg_catalog.pg_options_to_table(attfdwoptions) "
- "ORDER BY option_name"
- "), E',\n ') AS attfdwoptions,\n");
+ appendPQExpBuffer(q,
+ "SELECT\n"
+ " a.attrelid,\n"
+ " a.attnum,\n"
+ " a.attname,\n"
+ " a.attstattarget,\n"
+ " a.attstorage,\n"
+ " t.typstorage,\n"
+ " a.atthasdef,\n"
+ " a.attisdropped,\n"
+ " a.attlen,\n"
+ " a.attalign,\n"
+ " a.attislocal,\n"
+ " pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n"
+ " array_to_string(ARRAY(\n"
+ " SELECT x FROM unnest(a.attoptions) AS x %s\n"
+ " ), ', ') AS attoptions,\n"
+ " CASE WHEN a.attcollation <> t.typcollation"
+ " THEN a.attcollation ELSE 0 END AS attcollation,\n"
+ " pg_catalog.array_to_string(ARRAY("
+ " SELECT pg_catalog.quote_ident(option_name) || ' ' || "
+ " pg_catalog.quote_literal(option_value)"
+ " FROM pg_catalog.pg_options_to_table(attfdwoptions)"
+ " ORDER BY option_name"
+ " ), E',\n ') AS attfdwoptions,\n",
+ dopt->binary_upgrade ? "" : "WHERE x NOT LIKE 'dictid=%'");
/*
* Find out any NOT NULL markings for each column. In 18 and up we read
@@ -12158,12 +12300,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
char *typmodout;
char *typanalyze;
char *typsubscript;
+ char *typzstdsampling;
Oid typreceiveoid;
Oid typsendoid;
Oid typmodinoid;
Oid typmodoutoid;
Oid typanalyzeoid;
Oid typsubscriptoid;
+ Oid typzstdsamplingoid;
char *typcategory;
char *typispreferred;
char *typdelim;
@@ -12196,10 +12340,18 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
if (fout->remoteVersion >= 140000)
appendPQExpBufferStr(query,
"typsubscript, "
- "typsubscript::pg_catalog.oid AS typsubscriptoid ");
+ "typsubscript::pg_catalog.oid AS typsubscriptoid, ");
+ else
+ appendPQExpBufferStr(query,
+ "'-' AS typsubscript, 0 AS typsubscriptoid, ");
+
+ if (fout->remoteVersion >= 180000)
+ appendPQExpBufferStr(query,
+ "typzstdsampling, "
+ "typzstdsampling::pg_catalog.oid AS typzstdsamplingoid ");
else
appendPQExpBufferStr(query,
- "'-' AS typsubscript, 0 AS typsubscriptoid ");
+ "'-' AS typzstdsampling, 0 AS typzstdsamplingoid ");
appendPQExpBufferStr(query, "FROM pg_catalog.pg_type "
"WHERE oid = $1");
@@ -12224,12 +12376,14 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
typmodout = PQgetvalue(res, 0, PQfnumber(res, "typmodout"));
typanalyze = PQgetvalue(res, 0, PQfnumber(res, "typanalyze"));
typsubscript = PQgetvalue(res, 0, PQfnumber(res, "typsubscript"));
+ typzstdsampling = PQgetvalue(res, 0, PQfnumber(res, "typzstdsampling"));
typreceiveoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typreceiveoid")));
typsendoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsendoid")));
typmodinoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodinoid")));
typmodoutoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typmodoutoid")));
typanalyzeoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typanalyzeoid")));
typsubscriptoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typsubscriptoid")));
+ typzstdsamplingoid = atooid(PQgetvalue(res, 0, PQfnumber(res, "typzstdsamplingoid")));
typcategory = PQgetvalue(res, 0, PQfnumber(res, "typcategory"));
typispreferred = PQgetvalue(res, 0, PQfnumber(res, "typispreferred"));
typdelim = PQgetvalue(res, 0, PQfnumber(res, "typdelim"));
@@ -12285,7 +12439,8 @@ dumpBaseType(Archive *fout, const TypeInfo *tyinfo)
appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout);
if (OidIsValid(typanalyzeoid))
appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze);
-
+ if (OidIsValid(typzstdsamplingoid))
+ appendPQExpBuffer(q, ",\n ZSTD_SAMPLING = %s", typzstdsampling);
if (strcmp(typcollatable, "t") == 0)
appendPQExpBufferStr(q, ",\n COLLATABLE = true");
@@ -17542,6 +17697,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
case 'l':
cmname = "lz4";
break;
+ case 'z':
+ cmname = "zstd";
+ break;
default:
cmname = NULL;
break;
diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c
index 18c2d652bb6..38cae3ab0bd 100644
--- a/src/bin/pg_upgrade/check.c
+++ b/src/bin/pg_upgrade/check.c
@@ -14,6 +14,7 @@
#include "fe_utils/string_utils.h"
#include "pg_upgrade.h"
#include "common/unicode_version.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
static void check_new_cluster_is_empty(void);
static void check_is_install_user(ClusterInfo *cluster);
@@ -916,12 +917,29 @@ check_new_cluster_is_empty(void)
for (relnum = 0; relnum < rel_arr->nrels;
relnum++)
{
- /* pg_largeobject and its index should be skipped */
- if (strcmp(rel_arr->rels[relnum].nspname, "pg_catalog") != 0)
- pg_fatal("New cluster database \"%s\" is not empty: found relation \"%s.%s\"",
- new_cluster.dbarr.dbs[dbnum].db_name,
- rel_arr->rels[relnum].nspname,
- rel_arr->rels[relnum].relname);
+ const char *nspname = rel_arr->rels[relnum].nspname;
+ const char *relname = rel_arr->rels[relnum].relname;
+ Oid relOid = rel_arr->rels[relnum].reloid;
+
+ /**
+ * Allow all objects in pg_catalog
+ * pg_largeobject, pg_zstd_dictionaries and its index should be skipped.
+ */
+
+ if (strcmp(nspname, "pg_catalog") == 0)
+ continue;
+
+ /**
+ * Allow the specific toast objects for pg_zstd_dictionaries:
+ * fixed OIDs should be 9947 (toast table) or 9948 (toast index).
+ */
+ if (strcmp(nspname, "pg_toast") == 0 && (relOid == PgZstdDictionariesToastTable || relOid == PgZstdDictionariesToastIndex))
+ continue;
+
+ pg_fatal("New cluster database \"%s\" is not empty: found relation \"%s.%s\"",
+ new_cluster.dbarr.dbs[dbnum].db_name,
+ nspname,
+ relname);
}
}
}
diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c
index 4b7a56f5b3b..b38c125c77a 100644
--- a/src/bin/pg_upgrade/info.c
+++ b/src/bin/pg_upgrade/info.c
@@ -498,7 +498,7 @@ get_rel_infos_query(void)
" 'binary_upgrade', 'pg_toast') AND "
" c.oid >= %u::pg_catalog.oid) OR "
" (n.nspname = 'pg_catalog' AND "
- " relname IN ('pg_largeobject') ))), ",
+ " relname IN ('pg_largeobject', 'pg_zstd_dictionaries') ))), ",
(user_opts.transfer_mode == TRANSFER_MODE_SWAP) ?
", " CppAsString2(RELKIND_SEQUENCE) : "",
FirstNormalObjectId);
--
2.47.1
[application/octet-stream] v11-0004-Dependency-tracking-mechanism-to-track-compresse.patch (11.5K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/6-v11-0004-Dependency-tracking-mechanism-to-track-compresse.patch)
download | inline diff:
From 371865589fe3e456114caff5e1109e273d18c25f Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:50:07 +0000
Subject: [PATCH v11 4/7] Dependency tracking mechanism to track compressed
datum leaks to unrelated tables
---
src/backend/catalog/dependency.c | 105 +++++++++++++++++++++
src/backend/catalog/pg_zstd_dictionaries.c | 61 ++++++++++++
src/backend/commands/createas.c | 16 +---
src/backend/commands/prepare.c | 10 +-
src/backend/executor/nodeModifyTable.c | 5 +-
src/include/catalog/dependency.h | 2 +
src/include/catalog/pg_proc.dat | 5 +
src/include/commands/createas.h | 12 +++
8 files changed, 202 insertions(+), 14 deletions(-)
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index 0ea61ed1dae..c9a647d62ca 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -2838,3 +2838,108 @@ DeleteInitPrivs(const ObjectAddress *object)
table_close(relation, RowExclusiveLock);
}
+
+/*
+ * inheritZstdDictionaryDependencies - Record dictionary dependencies for a destination table.
+ *
+ * This function receives a list of relation OIDs. For each relation OID, it scans
+ * pg_depend to collect all associated dictids and then creates
+ * equivalent dependency entries for the destination table.
+ */
+void
+inheritZstdDictionaryDependencies(List *relationOids, Oid destRelid)
+{
+ List *relids;
+ List *src_dictids = NIL;
+ List *dest_dictids = NIL;
+ List *new_dictids;
+ ListCell *lc;
+ Relation depRel;
+ ScanKeyData skey[2];
+ SysScanDesc scan;
+ HeapTuple tup;
+
+ /* Build and deduplicate the list of all relation OIDs including destRelid */
+ relids = list_copy(relationOids);
+ relids = lappend_oid(relids, destRelid);
+ list_sort(relids, list_oid_cmp);
+ list_deduplicate_oid(relids);
+
+ /** Early exit if only destination relation is present
+ * During pg upgrade, dictionaries in the database are copied explicitly and their dependencies too.
+ */
+ if (list_length(relids) == 1 || IsBinaryUpgrade)
+ {
+ list_free(relids);
+ return;
+ }
+
+ depRel = table_open(DependRelationId, AccessShareLock);
+
+ /* Collect source and destination dictionaries from dependency table */
+ foreach(lc, relids)
+ {
+ Oid relOid = lfirst_oid(lc);
+
+ /* Initialize scan keys for current relation */
+ ScanKeyInit(&skey[0], Anum_pg_depend_classid,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(RelationRelationId));
+ ScanKeyInit(&skey[1], Anum_pg_depend_objid,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(relOid));
+
+ scan = systable_beginscan(depRel, DependDependerIndexId, true,
+ NULL, 2, skey);
+
+ while (HeapTupleIsValid(tup = systable_getnext(scan)))
+ {
+ Form_pg_depend dep = (Form_pg_depend) GETSTRUCT(tup);
+
+ /* Interested only in Zstd dictionary dependencies */
+ if (dep->refclassid != ZstdDictionariesRelationId)
+ continue;
+
+ if (dep->objid == destRelid)
+ dest_dictids = list_append_unique_oid(dest_dictids, dep->refobjid);
+ else
+ src_dictids = list_append_unique_oid(src_dictids, dep->refobjid);
+ }
+
+ systable_endscan(scan);
+ }
+
+ table_close(depRel, AccessShareLock);
+
+ /*
+ * Identify dictionaries from sources not already referenced by
+ * destination
+ */
+ new_dictids = list_difference_oid(src_dictids, dest_dictids);
+
+ /* Add new dictionary dependencies to the destination table if necessary */
+ if (new_dictids)
+ {
+ ObjectAddress depender;
+ ObjectAddresses *referenced = new_object_addresses();
+
+ ObjectAddressSet(depender, RelationRelationId, destRelid);
+
+ foreach(lc, new_dictids)
+ {
+ ObjectAddress dictObj;
+
+ ObjectAddressSet(dictObj, ZstdDictionariesRelationId, lfirst_oid(lc));
+ add_exact_object_address(&dictObj, referenced);
+ }
+
+ record_object_address_dependencies(&depender, referenced, DEPENDENCY_NORMAL);
+ free_object_addresses(referenced);
+ }
+
+ /* Clean up temporary lists */
+ list_free(relids);
+ list_free(src_dictids);
+ list_free(dest_dictids);
+ list_free(new_dictids);
+}
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
index 58964a600a3..5ae8ed71e48 100644
--- a/src/backend/catalog/pg_zstd_dictionaries.c
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -507,6 +507,67 @@ build_zstd_dict_for_attribute(PG_FUNCTION_ARGS)
#endif
}
+Datum
+cleanup_unused_zstd_dictionaries(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_INT32(cleanup_unused_zstd_dictionaries_internal());
+}
+
+static int
+cleanup_unused_zstd_dictionaries_internal(void)
+{
+ Relation dictRel,
+ depRel;
+ SysScanDesc dictScan,
+ depScan;
+ HeapTuple tuple;
+ List *used_dictids = NIL;
+ int dropped_count = 0;
+ ScanKeyData depKey;
+
+ /* Open necessary catalog relations */
+ dictRel = table_open(ZstdDictionariesRelationId, ShareRowExclusiveLock);
+ depRel = table_open(DependRelationId, AccessShareLock);
+
+ /* Find dictionary OIDs with dependencies */
+ ScanKeyInit(&depKey,
+ Anum_pg_depend_refclassid,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(ZstdDictionariesRelationId));
+
+ depScan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, 1, &depKey);
+ while ((tuple = systable_getnext(depScan)) != NULL)
+ {
+ Form_pg_depend dep = (Form_pg_depend) GETSTRUCT(tuple);
+
+ used_dictids = list_append_unique_oid(used_dictids, dep->refobjid);
+ }
+ systable_endscan(depScan);
+
+ /* Drop unused dictionaries */
+ dictScan = systable_beginscan(dictRel, InvalidOid, false, NULL, 0, NULL);
+ while ((tuple = systable_getnext(dictScan)) != NULL)
+ {
+ Oid dictid = ((Form_pg_zstd_dictionaries) GETSTRUCT(tuple))->dictid;
+
+ if (!list_member_oid(used_dictids, dictid))
+ {
+ ObjectAddress dictAddr;
+
+ ObjectAddressSet(dictAddr, ZstdDictionariesRelationId, dictid);
+ performDeletion(&dictAddr, DROP_RESTRICT, 0);
+ dropped_count++;
+ }
+ }
+ systable_endscan(dictScan);
+
+ /* Close catalog relations */
+ table_close(depRel, NoLock);
+ table_close(dictRel, NoLock);
+
+ return dropped_count;
+}
+
/*
* get_zstd_dict - Fetches the ZSTD dictionary from the catalog
*
diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c
index 0a4155773eb..2361ad77db4 100644
--- a/src/backend/commands/createas.c
+++ b/src/backend/commands/createas.c
@@ -47,18 +47,7 @@
#include "utils/lsyscache.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
-
-typedef struct
-{
- DestReceiver pub; /* publicly-known function pointers */
- IntoClause *into; /* target relation specification */
- /* These fields are filled by intorel_startup: */
- Relation rel; /* relation to write to */
- ObjectAddress reladdr; /* address of rel, for ExecCreateTableAs */
- CommandId output_cid; /* cmin to insert in output tuples */
- int ti_options; /* table_tuple_insert performance options */
- BulkInsertState bistate; /* bulk insert state */
-} DR_intorel;
+#include "catalog/dependency.h"
/* utility functions for CTAS definition creation */
static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into);
@@ -352,6 +341,9 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
/* get object address that intorel_startup saved for us */
address = ((DR_intorel *) dest)->reladdr;
+ /* Inherit zstd dictionary dependencies */
+ inheritZstdDictionaryDependencies(plan->relationOids, address.objectId);
+
/* and clean up */
ExecutorFinish(queryDesc);
ExecutorEnd(queryDesc);
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index bf7d2b2309f..a867f4d7711 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -36,7 +36,7 @@
#include "utils/builtins.h"
#include "utils/snapmgr.h"
#include "utils/timestamp.h"
-
+#include "catalog/dependency.h"
/*
* The hash table in which prepared queries are stored. This is
@@ -161,6 +161,7 @@ ExecuteQuery(ParseState *pstate,
char *query_string;
int eflags;
long count;
+ List *relationOids = NIL;
/* Look it up in the hash table */
entry = FetchPreparedStatement(stmt->name, true);
@@ -242,7 +243,10 @@ ExecuteQuery(ParseState *pstate,
if (intoClause->skipData)
count = 0;
else
+ {
count = FETCH_ALL;
+ relationOids = pstmt->relationOids;
+ }
}
else
{
@@ -258,6 +262,10 @@ ExecuteQuery(ParseState *pstate,
(void) PortalRun(portal, count, false, dest, dest, qc);
+ /* Inherit zstd dictionary dependencies */
+ if (intoClause && count != 0)
+ inheritZstdDictionaryDependencies(relationOids, ((DR_intorel *) dest)->reladdr.objectId);
+
PortalDrop(portal, false);
if (estate)
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 333cbf78343..e6847fd31c7 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -69,7 +69,7 @@
#include "utils/datum.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
-
+#include "catalog/dependency.h"
typedef struct MTTargetRelLookup
{
@@ -4416,6 +4416,9 @@ ExecModifyTable(PlanState *pstate)
if (estate->es_insert_pending_result_relations != NIL)
ExecPendingInserts(estate);
+ /* Inherit zstd dictionary dependencies */
+ inheritZstdDictionaryDependencies(estate->es_plannedstmt->relationOids, RelationGetRelid(resultRelInfo->ri_RelationDesc));
+
/*
* We're done, but fire AFTER STATEMENT triggers before exiting.
*/
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index 0ea7ccf5243..a1e91f2c8f1 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -225,4 +225,6 @@ extern void shdepDropOwned(List *roleids, DropBehavior behavior);
extern void shdepReassignOwned(List *roleids, Oid newrole);
+extern void inheritZstdDictionaryDependencies(List *relationOids, Oid destRelid);
+
#endif /* DEPENDENCY_H */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7d2286850dc..c98e9dca653 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -12592,6 +12592,11 @@
proargtypes => 'text int4', proparallel => 'u',
prosrc => 'build_zstd_dict_for_attribute' },
+{ oid => '9246', descr => 'cleanup unused dictionaries.',
+ proname => 'cleanup_unused_zstd_dictionaries', provolatile => 'v', prorettype => 'int4',
+ proargtypes => '', proparallel => 'u',
+ prosrc => 'cleanup_unused_zstd_dictionaries' },
+
{ oid => '9247', descr => 'ZSTD standard sampling for jsonb',
proname => 'std_zstd_sampling_for_jsonb', provolatile => 'v', prorettype => 'bool',
proargtypes => 'internal internal',
diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h
index 90612ebbb0e..2ee78652f28 100644
--- a/src/include/commands/createas.h
+++ b/src/include/commands/createas.h
@@ -19,7 +19,19 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
#include "utils/queryenvironment.h"
+#include "access/heapam.h"
+typedef struct
+{
+ DestReceiver pub; /* publicly-known function pointers */
+ IntoClause *into; /* target relation specification */
+ /* These fields are filled by intorel_startup: */
+ Relation rel; /* relation to write to */
+ ObjectAddress reladdr; /* address of rel, for ExecCreateTableAs */
+ CommandId output_cid; /* cmin to insert in output tuples */
+ int ti_options; /* table_tuple_insert performance options */
+ BulkInsertState bistate; /* bulk insert state */
+} DR_intorel;
extern ObjectAddress ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt,
ParamListInfo params, QueryEnvironment *queryEnv,
--
2.47.1
[application/octet-stream] v11-0001-varattrib_4b-changes-and-macros-update-needed-to.patch (4.8K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/7-v11-0001-varattrib_4b-changes-and-macros-update-needed-to.patch)
download | inline diff:
From e7e8aa07af0466f4a05d7df1c002596b69563f4a Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:07:00 +0000
Subject: [PATCH v11 1/7] varattrib_4b changes and macros update needed to
support zstd dictionary based compression.
---
src/include/varatt.h | 57 ++++++++++++++++++++++++++++++++++++++------
1 file changed, 50 insertions(+), 7 deletions(-)
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 2e8564d4998..ba06250b6ce 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -42,8 +42,9 @@ typedef struct varatt_external
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
*/
-#define VARLENA_EXTSIZE_BITS 30
-#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTSIZE_BITS 30
+#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
+#define VARLENA_EXTENDED_COMPRESSION_FLAG 0x3
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
@@ -122,6 +123,14 @@ typedef union
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
+ struct
+ {
+ uint32 va_header;
+ uint32 va_tcinfo;
+ uint32 va_cmp_alg;
+ uint32 va_cmp_dictid;
+ char va_data[FLEXIBLE_ARRAY_MEMBER];
+ } va_compressed_ext;
} varattrib_4b;
typedef struct
@@ -242,7 +251,14 @@ typedef struct
#endif /* WORDS_BIGENDIAN */
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
-#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
+/*
+ * If va_tcinfo >> VARLENA_EXTSIZE_BITS == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * use va_compressed_ext; otherwise, use the va_compressed.
+ */
+#define VARDATA_4B_C(PTR) \
+( (((varattrib_4b *)(PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG \
+ ? ((varattrib_4b *)(PTR))->va_compressed_ext.va_data \
+ : ((varattrib_4b *)(PTR))->va_compressed.va_data )
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
@@ -252,6 +268,7 @@ typedef struct
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
+#define VARHDRSZ_COMPRESSED_EXT offsetof(varattrib_4b, va_compressed_ext.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
@@ -327,8 +344,20 @@ typedef struct
/* Decompressed size and compression method of a compressed-in-line Datum */
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
+/*
+ * - "Extended" format is indicated by (va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG
+ * - For the non-extended formats, the method code is stored in the top bits of va_tcinfo.
+ * - In the extended format, the method code is stored in va_cmp_alg instead.
+ */
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
- (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
+( ((((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((varattrib_4b *) (PTR))->va_compressed_ext.va_cmp_alg) \
+ : ( (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo) >> VARLENA_EXTSIZE_BITS))
+
+#define VARDATA_COMPRESSED_GET_DICTID(PTR) \
+ ( ((((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((varattrib_4b *) (PTR))->va_compressed_ext.va_cmp_dictid) \
+ : InvalidDictId)
/* Same for external Datums; but note argument is a struct varatt_external */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
@@ -338,10 +367,24 @@ typedef struct
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
+ /* If desired, keep or expand the Assert checks for known methods: */ \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == TOAST_ZSTD_COMPRESSION_ID); \
+ if ((cm) < TOAST_ZSTD_COMPRESSION_ID) \
+ { \
+ /* Store the actual method in va_extinfo */ \
+ (toast_pointer).va_extinfo = (uint32)(len) \
+ | ((uint32)(cm) << VARLENA_EXTSIZE_BITS); \
+ } \
+ else \
+ { \
+ /* Store VARLENA_EXTENDED_COMPRESSION_FLAG in the top bits, \
+ meaning "extended" method. */ \
+ (toast_pointer).va_extinfo = (uint32)(len) | \
+ ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG \
+ << VARLENA_EXTSIZE_BITS); \
+ } \
} while (0)
/*
base-commit: 7c872849407730fa01e2c13b2d47483bc3ff6e7e
--
2.47.1
[application/octet-stream] v11-0002-Zstd-compression-and-decompression-routines-incl.patch (51.7K, ../../CAFAfj_Gb-1HxZ303MkX5Z8skzc6DAg0Ygc_nRqop3LB-PB2bnQ@mail.gmail.com/8-v11-0002-Zstd-compression-and-decompression-routines-incl.patch)
download | inline diff:
From a72c8e8c62e4d5bb8b9de40faf85a8a3422f87dc Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 14 Apr 2025 21:30:03 +0000
Subject: [PATCH v11 2/7] Zstd compression and decompression routines,
including new catalog table pg_zstd_dictionaries to store zstd dictionaries.
---
contrib/amcheck/verify_heapam.c | 1 +
doc/src/sgml/catalogs.sgml | 55 +++
src/backend/access/brin/brin_tuple.c | 11 +-
src/backend/access/common/detoast.c | 12 +-
src/backend/access/common/indextuple.c | 5 +-
src/backend/access/common/reloptions.c | 36 +-
src/backend/access/common/toast_compression.c | 328 +++++++++++++++++-
src/backend/access/common/toast_internals.c | 49 ++-
src/backend/access/table/toast_helper.c | 15 +-
src/backend/catalog/Makefile | 3 +-
src/backend/catalog/aclchk.c | 2 +
src/backend/catalog/catalog.c | 4 +
src/backend/catalog/dependency.c | 2 +
src/backend/catalog/meson.build | 1 +
src/backend/catalog/objectaddress.c | 76 ++++
src/backend/catalog/pg_zstd_dictionaries.c | 56 +++
src/backend/commands/dropcmds.c | 1 +
src/backend/commands/event_trigger.c | 2 +
src/backend/commands/seclabel.c | 1 +
src/backend/utils/adt/varlena.c | 3 +
src/backend/utils/misc/guc_tables.c | 3 +
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/bin/psql/describe.c | 5 +-
src/bin/psql/tab-complete.in.c | 4 +-
src/include/access/toast_compression.h | 26 +-
src/include/access/toast_helper.h | 2 +
src/include/access/toast_internals.h | 44 ++-
src/include/catalog/Makefile | 3 +-
src/include/catalog/meson.build | 1 +
src/include/catalog/pg_zstd_dictionaries.h | 48 +++
src/include/nodes/parsenodes.h | 1 +
src/include/utils/attoptcache.h | 6 +
src/test/regress/expected/compression.out | 5 +-
src/test/regress/expected/compression_1.out | 3 +
src/test/regress/sql/compression.sql | 1 +
src/tools/pgindent/typedefs.list | 2 +
36 files changed, 774 insertions(+), 45 deletions(-)
create mode 100644 src/backend/catalog/pg_zstd_dictionaries.c
create mode 100644 src/include/catalog/pg_zstd_dictionaries.h
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 2152d8ee577..74991373da9 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1792,6 +1792,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* List of all valid compression method IDs */
case TOAST_PGLZ_COMPRESSION_ID:
case TOAST_LZ4_COMPRESSION_ID:
+ case TOAST_ZSTD_COMPRESSION_ID:
valid = true;
break;
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index cbd4e40a320..ab44031d726 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -369,6 +369,12 @@
<entry><link linkend="catalog-pg-user-mapping"><structname>pg_user_mapping</structname></link></entry>
<entry>mappings of users to foreign servers</entry>
</row>
+
+ <row>
+ <entry><link linkend="catalog-pg-zstd-dictionaries"><structname>pg_zstd_dictionaries</structname></link></entry>
+ <entry>Zstandard dictionaries</entry>
+ </row>
+
</tbody>
</tgroup>
</table>
@@ -9779,4 +9785,53 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
</table>
</sect1>
+
+<sect1 id="catalog-pg-zstd-dictionaries">
+ <title><structname>pg_zstd_dictionaries</structname></title>
+
+ <indexterm zone="catalog-pg-zstd-dictionaries">
+ <primary>pg_zstd_dictionaries</primary>
+ </indexterm>
+
+ <para>
+ The catalog <structname>pg_zstd_dictionaries</structname> maintains the dictionaries essential for Zstandard compression and decompression.
+ </para>
+
+ <table>
+ <title><structname>pg_zstd_dictionaries</structname> Columns</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">Column Type</para>
+ <para>Description</para>
+ </entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dictid</structfield> <type>oid</type>
+ </para>
+ <para>
+ Dictionary identifier; a non-null OID that uniquely identifies a dictionary.
+ </para>
+ </entry>
+ </row>
+ <row>
+ <entry role="catalog_table_entry">
+ <para role="column_definition">
+ <structfield>dict</structfield> <type>bytea</type>
+ </para>
+ <para>
+ Variable-length field containing the zstd dictionary data. This field must not be null.
+ </para>
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+</sect1>
+
</chapter>
diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c
index 861f397e6db..56099acaa89 100644
--- a/src/backend/access/brin/brin_tuple.c
+++ b/src/backend/access/brin/brin_tuple.c
@@ -222,10 +222,13 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
atttype->typstorage == TYPSTORAGE_MAIN))
{
Datum cvalue;
- char compression;
+ CompressionInfo cmp;
+
Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
keyno);
+ setup_compression_info(&cmp, att);
+
/*
* If the BRIN summary and indexed attribute use the same data
* type and it has a valid compression method, we can use the
@@ -233,11 +236,11 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
* default method.
*/
if (att->atttypid == atttype->type_id)
- compression = att->attcompression;
+ cmp.cmethod = att->attcompression;
else
- compression = InvalidCompressionMethod;
+ cmp.cmethod = InvalidCompressionMethod;
- cvalue = toast_compress_datum(value, compression);
+ cvalue = toast_compress_datum(value, cmp);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 62651787742..b57a9f024c7 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr,
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or Zstandard data, we'll have to
+ * fetch the whole thing, because there doesn't seem to be an API
+ * call to determine how much compressed data we need to be sure
+ * of being able to decompress the required slice.
*/
if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
TOAST_PGLZ_COMPRESSION_ID)
@@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_ZSTD_COMPRESSION_ID:
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index 1986b943a28..cbb57d17799 100644
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -123,9 +123,10 @@ index_form_tuple_context(TupleDesc tupleDescriptor,
att->attstorage == TYPSTORAGE_MAIN))
{
Datum cvalue;
+ CompressionInfo cmp;
- cvalue = toast_compress_datum(untoasted_values[i],
- att->attcompression);
+ setup_compression_info(&cmp, att);
+ cvalue = toast_compress_datum(untoasted_values[i], cmp);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 46c1dce222d..345e3dcdb2c 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -34,6 +34,7 @@
#include "utils/guc.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "access/toast_compression.h"
/*
* Contents of pg_class.reloptions
@@ -381,7 +382,26 @@ static relopt_int intRelOpts[] =
},
-1, 0, 1024
},
-
+ {
+ {
+ "zstd_dict_size",
+ "Max dict size for zstd",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_DICT_SIZE, 0, 112640 /* Max dict size(110 KB), 0
+ * indicates don't use dictionary
+ * for compression */
+ },
+ {
+ {
+ "zstd_level",
+ "Set column's ZSTD compression level",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_LEVEL, MIN_ZSTD_LEVEL, MAX_ZSTD_LEVEL
+ },
/* list terminator */
{{NULL}}
};
@@ -470,6 +490,15 @@ static relopt_real realRelOpts[] =
},
0, -1.0, DBL_MAX
},
+ {
+ {
+ "dictid",
+ "Current dictid for column",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ InvalidDictId, InvalidDictId, UINT32_MAX
+ },
{
{
"vacuum_cleanup_index_scale_factor",
@@ -2097,7 +2126,10 @@ attribute_reloptions(Datum reloptions, bool validate)
{
static const relopt_parse_elt tab[] = {
{"n_distinct", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct)},
- {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)}
+ {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)},
+ {"dictid", RELOPT_TYPE_REAL, offsetof(AttributeOpts, dictid)},
+ {"zstd_dict_size", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_dict_size)},
+ {"zstd_level", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_level)},
};
return (bytea *) build_reloptions(reloptions, validate,
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 21f2f4af97e..3b028993f0b 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -17,19 +17,41 @@
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
+
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"
+#include "catalog/pg_zstd_dictionaries.h"
+#include "access/toast_internals.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
-#define NO_LZ4_SUPPORT() \
+#ifdef USE_ZSTD
+static ZSTD_CCtx *ZstdCompressionCtx = NULL;
+static ZSTD_CDict * ZstdCompressionCtxCDict = NULL;
+static Oid ZstdCompressionCtxDictID = InvalidDictId;
+
+static ZSTD_DCtx *ZstdDecompressionCtx = NULL;
+static ZSTD_DDict * ZstdDecompressionCtxDDict = NULL;
+static Oid ZstdDecompressionCtxDictID = InvalidDictId;
+
+#define ZSTD_CHECK_ERROR(zstd_ret, msg) \
+ do { \
+ if (ZSTD_isError(zstd_ret)) \
+ ereport(ERROR, (errmsg("%s: %s", (msg), ZSTD_getErrorName(zstd_ret)))); \
+ } while (0)
+#endif
+
+#define COMPRESSION_METHOD_NOT_SUPPORTED(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with %s support.", method)))
/*
* Compress a varlena using PGLZ.
@@ -139,7 +161,7 @@ struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 valsize;
@@ -182,7 +204,7 @@ struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -215,7 +237,7 @@ struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -266,7 +288,13 @@ toast_get_compression_id(struct varlena *attr)
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) >= TOAST_ZSTD_COMPRESSION_ID)
+ {
+ struct varlena *compressed_attr = detoast_external_attr(attr);
+
+ cmid = TOAST_COMPRESS_METHOD(compressed_attr);
+ }
+ else
cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
}
else if (VARATT_IS_COMPRESSED(attr))
@@ -289,10 +317,17 @@ CompressionNameToMethod(const char *compression)
else if (strcmp(compression, "lz4") == 0)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd") == 0)
+ {
+#ifndef USE_ZSTD
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd");
+#endif
+ return TOAST_ZSTD_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -309,8 +344,285 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_COMPRESSION:
+ return "zstd";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
}
}
+
+/* Compress datum using ZSTD with optional dictionary (using cdict) */
+struct varlena *
+zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level)
+{
+#ifdef USE_ZSTD
+ uint32 valsize = VARSIZE_ANY_EXHDR(value);
+ size_t max_size = ZSTD_compressBound(valsize);
+ struct varlena *compressed;
+ void *dest;
+ size_t cmp_size,
+ ret;
+
+ /* Create the session CCtx if it hasn't been yet */
+ if (ZstdCompressionCtx == NULL)
+ {
+ ZstdCompressionCtx = ZSTD_createCCtx();
+ if (!ZstdCompressionCtx)
+ ereport(ERROR, (errmsg("could not create ZSTD_CCtx")));
+ ZstdCompressionCtxDictID = InvalidDictId;
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_CCtx_reset(ZstdCompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD CCtx");
+
+ /* Set compression level */
+ ret = ZSTD_CCtx_setParameter(ZstdCompressionCtx, ZSTD_c_compressionLevel, zstd_level);
+ ZSTD_CHECK_ERROR(ret, "failed to set ZSTD compression level");
+
+ /* Check and update dictionary if changed */
+ if (ZstdCompressionCtxDictID != dictid)
+ {
+ /* If there's a previous dictionary, detach and free it */
+ if (ZstdCompressionCtxCDict)
+ {
+ ZSTD_freeCDict(ZstdCompressionCtxCDict);
+ ZstdCompressionCtxCDict = NULL;
+ }
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+ ZSTD_CDict *cdict = ZSTD_createCDict(dict_buffer, dict_size, zstd_level);
+
+ pfree(dict_bytea);
+
+ if (!cdict)
+ ereport(ERROR, (errmsg("Failed to create ZSTD compression dictionary")));
+
+ ret = ZSTD_CCtx_refCDict(ZstdCompressionCtx, cdict);
+ ZSTD_CHECK_ERROR(ret, "failed to load ZSTD dictionary");
+
+ ZstdCompressionCtxCDict = cdict;
+ }
+ else
+ {
+ /* Unload any previously used dictionary by passing NULL */
+ ret = ZSTD_CCtx_refCDict(ZstdCompressionCtx, NULL);
+ ZSTD_CHECK_ERROR(ret, "failed to unload ZSTD dictionary");
+ }
+
+ ZstdCompressionCtxDictID = dictid;
+ }
+
+ /* Allocate space for the compressed varlena (header + data) */
+ compressed = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED_EXT);
+ dest = (char *) compressed + VARHDRSZ_COMPRESSED_EXT;
+
+ cmp_size = ZSTD_compress2(ZstdCompressionCtx,
+ dest,
+ max_size,
+ VARDATA_ANY(value),
+ valsize);
+
+ if (ZSTD_isError(cmp_size))
+ {
+ pfree(compressed);
+ ZSTD_CHECK_ERROR(cmp_size, "ZSTD compression failed");
+ }
+
+ /*
+ * If compression did not reduce size, return NULL so that the
+ * uncompressed data is stored
+ */
+ if (cmp_size > valsize)
+ {
+ pfree(compressed);
+ return NULL;
+ }
+
+ /* Set the compressed size in the varlena header */
+ SET_VARSIZE_COMPRESSED(compressed, cmp_size + VARHDRSZ_COMPRESSED_EXT);
+ return compressed;
+
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd");
+ return NULL;
+#endif
+}
+
+/* Decompression routine */
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifdef USE_ZSTD
+ uint32 actual_size_exhdr = VARDATA_COMPRESSED_GET_EXTSIZE(value);
+ uint32 cmp_size_exhdr = VARSIZE_4B(value) - VARHDRSZ_COMPRESSED_EXT;
+ Oid dictid = (Oid) VARDATA_COMPRESSED_GET_DICTID(value);
+ struct varlena *result;
+ size_t uncmp_size,
+ ret;
+
+ if (ZstdDecompressionCtx == NULL)
+ {
+ ZstdDecompressionCtx = ZSTD_createDCtx();
+ if (!ZstdDecompressionCtx)
+ ereport(ERROR, (errmsg("could not create ZSTD_DCtx")));
+ ZstdDecompressionCtxDictID = InvalidDictId;
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_DCtx_reset(ZstdDecompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD DCtx");
+
+ if (ZstdDecompressionCtxDictID != dictid)
+ {
+ if (ZstdDecompressionCtxDDict)
+ {
+ ZSTD_freeDDict(ZstdDecompressionCtxDDict);
+ ZstdDecompressionCtxDDict = NULL;
+ }
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+ ZSTD_DDict *ddict = ZSTD_createDDict(dict_buffer, dict_size);
+
+ pfree(dict_bytea);
+
+ if (!ddict)
+ ereport(ERROR, (errmsg("Failed to create ZSTD decompression dictionary")));
+
+ ret = ZSTD_DCtx_refDDict(ZstdDecompressionCtx, ddict);
+ ZSTD_CHECK_ERROR(ret, "failed to load ZSTD dictionary");
+
+ ZstdDecompressionCtxDDict = ddict;
+ }
+ else
+ {
+ /* Unload any previously used dictionary by passing NULL */
+ ret = ZSTD_DCtx_refDDict(ZstdDecompressionCtx, NULL);
+ ZSTD_CHECK_ERROR(ret, "failed to unload ZSTD dictionary");
+ }
+
+ /* Update the tracked dictionary ID */
+ ZstdDecompressionCtxDictID = dictid;
+ }
+
+ /* Allocate space for the uncompressed data */
+ result = (struct varlena *) palloc(actual_size_exhdr + VARHDRSZ);
+
+ uncmp_size = ZSTD_decompressDCtx(ZstdDecompressionCtx,
+ VARDATA(result),
+ actual_size_exhdr,
+ VARDATA_4B_C(value),
+ cmp_size_exhdr);
+
+ if (ZSTD_isError(uncmp_size))
+ {
+ pfree(result);
+ ZSTD_CHECK_ERROR(uncmp_size, "ZSTD decompression failed");
+ }
+
+ /* Set final size in the varlena header */
+ SET_VARSIZE(result, uncmp_size + VARHDRSZ);
+ return result;
+
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd");
+ return NULL;
+#endif
+}
+
+/* Decompress a slice of the datum using the streaming API and optional dictionary */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+#ifdef USE_ZSTD
+ struct varlena *result;
+ ZSTD_inBuffer inBuf;
+ ZSTD_outBuffer outBuf;
+ Oid dictid = (Oid) VARDATA_COMPRESSED_GET_DICTID(value);
+ size_t ret;
+
+ if (ZstdDecompressionCtx == NULL)
+ {
+ ZstdDecompressionCtx = ZSTD_createDCtx();
+ if (!ZstdDecompressionCtx)
+ elog(ERROR, "could not create ZSTD_DCtx");
+ ZstdDecompressionCtxDictID = InvalidDictId;
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_DCtx_reset(ZstdDecompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD_DCtx");
+
+ if (ZstdDecompressionCtxDictID != dictid)
+ {
+ if (ZstdDecompressionCtxDDict)
+ {
+ ZSTD_freeDDict(ZstdDecompressionCtxDDict);
+ ZstdDecompressionCtxDDict = NULL;
+ }
+
+ if (dictid != InvalidDictId)
+ {
+ bytea *dict_bytea = get_zstd_dict(dictid);
+ const void *dict_buffer = VARDATA_ANY(dict_bytea);
+ uint32 dict_size = VARSIZE_ANY(dict_bytea) - VARHDRSZ;
+ ZSTD_DDict *ddict = ZSTD_createDDict(dict_buffer, dict_size);
+
+ pfree(dict_bytea);
+
+ if (!ddict)
+ ereport(ERROR, (errmsg("Failed to create ZSTD decompression dictionary")));
+
+ ret = ZSTD_DCtx_refDDict(ZstdDecompressionCtx, ddict);
+ ZSTD_CHECK_ERROR(ret, "failed to load ZSTD dictionary");
+
+ ZstdDecompressionCtxDDict = ddict;
+ }
+ else
+ {
+ /* Unload any previously used dictionary by passing NULL */
+ ret = ZSTD_DCtx_refDDict(ZstdDecompressionCtx, NULL);
+ ZSTD_CHECK_ERROR(ret, "failed to unload ZSTD dictionary");
+ }
+
+ /* Update the tracked dictionary ID */
+ ZstdDecompressionCtxDictID = dictid;
+ }
+
+ inBuf.src = (char *) value + VARHDRSZ_COMPRESSED_EXT;
+ inBuf.size = VARSIZE(value) - VARHDRSZ_COMPRESSED_EXT;
+ inBuf.pos = 0;
+
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+ outBuf.dst = (char *) result + VARHDRSZ;
+ outBuf.size = slicelength;
+ outBuf.pos = 0;
+
+ /* Common decompression loop */
+ while (inBuf.pos < inBuf.size && outBuf.pos < outBuf.size)
+ {
+ ret = ZSTD_decompressStream(ZstdDecompressionCtx, &outBuf, &inBuf);
+ if (ZSTD_isError(ret))
+ {
+ pfree(result);
+ ZSTD_CHECK_ERROR(ret, "zstd decompression failed");
+ }
+ }
+
+ Assert(outBuf.size == slicelength && outBuf.pos == slicelength);
+ SET_VARSIZE(result, outBuf.pos + VARHDRSZ);
+ return result;
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd");
+ return NULL;
+#endif
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 7d8be8346ce..46f0381b2de 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -25,6 +25,7 @@
#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
+#include "utils/attoptcache.h"
static bool toastrel_valueid_exists(Relation toastrel, Oid valueid);
static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
@@ -43,7 +44,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
* ----------
*/
Datum
-toast_compress_datum(Datum value, char cmethod)
+toast_compress_datum(Datum value, CompressionInfo cmp)
{
struct varlena *tmp = NULL;
int32 valsize;
@@ -55,13 +56,13 @@ toast_compress_datum(Datum value, char cmethod)
valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
/* If the compression method is not valid, use the current default */
- if (!CompressionMethodIsValid(cmethod))
- cmethod = default_toast_compression;
+ if (!CompressionMethodIsValid(cmp.cmethod))
+ cmp.cmethod = default_toast_compression;
/*
* Call appropriate compression routine for the compression method.
*/
- switch (cmethod)
+ switch (cmp.cmethod)
{
case TOAST_PGLZ_COMPRESSION:
tmp = pglz_compress_datum((const struct varlena *) value);
@@ -71,8 +72,12 @@ toast_compress_datum(Datum value, char cmethod)
tmp = lz4_compress_datum((const struct varlena *) value);
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_COMPRESSION:
+ tmp = zstd_compress_datum((const struct varlena *) value, cmp.dictid, cmp.zstd_level);
+ cmid = TOAST_ZSTD_COMPRESSION_ID;
+ break;
default:
- elog(ERROR, "invalid compression method %c", cmethod);
+ elog(ERROR, "invalid compression method %c", cmp.cmethod);
}
if (tmp == NULL)
@@ -90,9 +95,11 @@ toast_compress_datum(Datum value, char cmethod)
*/
if (VARSIZE(tmp) < valsize - 2)
{
+ Oid dictid = cmp.dictid;
+
/* successful compression */
Assert(cmid != TOAST_INVALID_COMPRESSION_ID);
- TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid);
+ TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid, dictid);
return PointerGetDatum(tmp);
}
else
@@ -654,3 +661,33 @@ get_toast_snapshot(void)
return &SnapshotToastData;
}
+
+void
+setup_compression_info(CompressionInfo * info, Form_pg_attribute att)
+{
+ info->cmethod = att->attcompression;
+ info->dictid = InvalidDictId;
+ info->zstd_level = DEFAULT_ZSTD_LEVEL;
+
+ if (att->attcompression == TOAST_ZSTD_COMPRESSION)
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ {
+ info->zstd_level = aopt->zstd_level;
+ /**
+ * If user marks zstd dict size as 0, then we don't use dict compression
+ * for this attribute.
+ */
+ if (aopt->zstd_dict_size != 0)
+ {
+ if (aopt->dictid < InvalidDictId || aopt->dictid > UINT32_MAX)
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("dictid is not in expected range")));
+ info->dictid = (Oid) aopt->dictid;
+ }
+ }
+ }
+}
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index b60fab0a4d2..f4b1cbe494e 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -51,10 +51,15 @@ toast_tuple_init(ToastTupleContext *ttc)
Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
struct varlena *old_value;
struct varlena *new_value;
+ CompressionInfo cmp;
+
+ setup_compression_info(&cmp, att);
ttc->ttc_attr[i].tai_colflags = 0;
ttc->ttc_attr[i].tai_oldexternal = NULL;
- ttc->ttc_attr[i].tai_compression = att->attcompression;
+ ttc->ttc_attr[i].tai_compression = cmp.cmethod;
+ ttc->ttc_attr[i].dictid = cmp.dictid;
+ ttc->ttc_attr[i].zstd_level = cmp.zstd_level;
if (ttc->ttc_oldvalues != NULL)
{
@@ -230,7 +235,13 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
Datum new_value;
ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
- new_value = toast_compress_datum(*value, attr->tai_compression);
+ CompressionInfo cmp = {
+ .cmethod = attr->tai_compression,
+ .dictid = attr->dictid,
+ .zstd_level = attr->zstd_level
+ };
+
+ new_value = toast_compress_datum(*value, cmp);
if (DatumGetPointer(new_value) != NULL)
{
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index c090094ed08..282afbcef5e 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -46,7 +46,8 @@ OBJS = \
pg_subscription.o \
pg_type.o \
storage.o \
- toasting.o
+ toasting.o \
+ pg_zstd_dictionaries.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 9ca8a88dc91..01d70ebc53e 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2771,6 +2771,7 @@ aclcheck_error(AclResult aclerr, ObjectType objtype,
case OBJECT_TSPARSER:
case OBJECT_TSTEMPLATE:
case OBJECT_USER_MAPPING:
+ case OBJECT_ZSTD_DICTIONARY:
elog(ERROR, "unsupported object type: %d", objtype);
}
@@ -2909,6 +2910,7 @@ aclcheck_error(AclResult aclerr, ObjectType objtype,
case OBJECT_TSPARSER:
case OBJECT_TSTEMPLATE:
case OBJECT_USER_MAPPING:
+ case OBJECT_ZSTD_DICTIONARY:
elog(ERROR, "unsupported object type: %d", objtype);
}
diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c
index a6edf614606..d89334f7e87 100644
--- a/src/backend/catalog/catalog.c
+++ b/src/backend/catalog/catalog.c
@@ -40,6 +40,7 @@
#include "catalog/pg_subscription.h"
#include "catalog/pg_tablespace.h"
#include "catalog/pg_type.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
#include "miscadmin.h"
#include "utils/fmgroids.h"
#include "utils/fmgrprotos.h"
@@ -381,6 +382,9 @@ IsPinnedObject(Oid classId, Oid objectId)
if (classId == DatabaseRelationId)
return false;
+ if (classId == ZstdDictionariesRelationId)
+ return false;
+
/*
* All other initdb-created objects are pinned. This is overkill (the
* system doesn't really depend on having every last weird datatype, for
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index 18316a3968b..0ea61ed1dae 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -65,6 +65,7 @@
#include "catalog/pg_ts_template.h"
#include "catalog/pg_type.h"
#include "catalog/pg_user_mapping.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
#include "commands/comment.h"
#include "commands/defrem.h"
#include "commands/event_trigger.h"
@@ -1464,6 +1465,7 @@ doDeletion(const ObjectAddress *object, int flags)
case EventTriggerRelationId:
case TransformRelationId:
case AuthMemRelationId:
+ case ZstdDictionariesRelationId:
DropObjectById(object);
break;
diff --git a/src/backend/catalog/meson.build b/src/backend/catalog/meson.build
index 1958ea9238a..8f0413189cb 100644
--- a/src/backend/catalog/meson.build
+++ b/src/backend/catalog/meson.build
@@ -34,6 +34,7 @@ backend_sources += files(
'pg_type.c',
'storage.c',
'toasting.c',
+ 'pg_zstd_dictionaries.c',
)
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index b63fd57dc04..ba3e52e5323 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -62,6 +62,7 @@
#include "catalog/pg_ts_template.h"
#include "catalog/pg_type.h"
#include "catalog/pg_user_mapping.h"
+#include "catalog/pg_zstd_dictionaries.h"
#include "commands/dbcommands.h"
#include "commands/defrem.h"
#include "commands/event_trigger.h"
@@ -636,6 +637,20 @@ static const ObjectPropertyType ObjectProperty[] =
OBJECT_USER_MAPPING,
false
},
+ {
+ "zstd dictionaries",
+ ZstdDictionariesRelationId,
+ ZstdDictidIndexId,
+ ZSTDDICTIDOID,
+ -1,
+ Anum_pg_zstd_dictionaries_dictid,
+ InvalidAttrNumber,
+ InvalidAttrNumber,
+ InvalidAttrNumber,
+ InvalidAttrNumber,
+ OBJECT_ZSTD_DICTIONARY,
+ true
+ },
};
/*
@@ -831,6 +846,9 @@ static const struct object_type_map
},
{
"statistics object", OBJECT_STATISTIC_EXT
+ },
+ {
+ "zstd dictionary", OBJECT_ZSTD_DICTIONARY
}
};
@@ -1127,6 +1145,26 @@ get_object_address(ObjectType objtype, Node *object,
missing_ok);
address.objectSubId = 0;
break;
+ case OBJECT_ZSTD_DICTIONARY:
+ {
+ Oid dictid = oidparse(object);
+ HeapTuple tuple = SearchSysCache1(ZSTDDICTIDOID, ObjectIdGetDatum(dictid));
+
+ if (!HeapTupleIsValid(tuple))
+ {
+ if (!missing_ok)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("zstd dictionary %u does not exist",
+ dictid)));
+ }
+ ReleaseSysCache(tuple);
+
+ address.classId = ZstdDictionariesRelationId;
+ address.objectId = dictid;
+ address.objectSubId = 0;
+ break;
+ }
/* no default, to let compiler warn about missing case */
}
@@ -2172,6 +2210,23 @@ pg_get_object_address(PG_FUNCTION_ARGS)
errmsg("large object OID may not be null")));
objnode = (Node *) makeFloat(TextDatumGetCString(elems[0]));
}
+ else if (type == OBJECT_ZSTD_DICTIONARY)
+ {
+ Datum *elems;
+ bool *nulls;
+ int nelems;
+
+ deconstruct_array_builtin(namearr, TEXTOID, &elems, &nulls, &nelems);
+ if (nelems != 1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("name list length must be exactly %d", 1)));
+ if (nulls[0])
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("zstd dictionary OID may not be null")));
+ objnode = (Node *) makeFloat(TextDatumGetCString(elems[0]));
+ }
else
{
name = textarray_to_strvaluelist(namearr);
@@ -2354,6 +2409,7 @@ pg_get_object_address(PG_FUNCTION_ARGS)
break;
}
case OBJECT_LARGEOBJECT:
+ case OBJECT_ZSTD_DICTIONARY:
/* already handled above */
break;
/* no default, to let compiler warn about missing case */
@@ -2557,6 +2613,7 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address,
case OBJECT_PUBLICATION_NAMESPACE:
case OBJECT_PUBLICATION_REL:
case OBJECT_USER_MAPPING:
+ case OBJECT_ZSTD_DICTIONARY:
/* These are currently not supported or don't make sense here. */
elog(ERROR, "unsupported object type: %d", (int) objtype);
break;
@@ -4067,7 +4124,26 @@ getObjectDescription(const ObjectAddress *object, bool missing_ok)
ReleaseSysCache(trfTup);
break;
}
+ case ZstdDictionariesRelationId:
+ {
+ HeapTuple htup;
+ Form_pg_zstd_dictionaries zstd;
+ htup = SearchSysCache1(ZSTDDICTIDOID, ObjectIdGetDatum(object->objectId));
+ if (!HeapTupleIsValid(htup))
+ {
+ if (!missing_ok)
+ elog(ERROR, "could not find tuple for dictid %u",
+ object->objectId);
+ break;
+ }
+
+ zstd = (Form_pg_zstd_dictionaries) GETSTRUCT(htup);
+ appendStringInfo(&buffer, _("Dictionary Id %d"), zstd->dictid);
+
+ ReleaseSysCache(htup);
+ break;
+ }
default:
elog(ERROR, "unsupported object class: %u", object->classId);
}
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
new file mode 100644
index 00000000000..d5e965c34d0
--- /dev/null
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -0,0 +1,56 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_zstd_dictionaries.c
+ * routines to support manipulation of the pg_zstd_dictionaries relation
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/catalog/pg_zstd_dictionaries.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "catalog/pg_zstd_dictionaries.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+#include "utils/syscache.h"
+#include "varatt.h"
+
+/*
+ * get_zstd_dict - Fetches the ZSTD dictionary from the catalog
+ *
+ * dictid: The Oid of the dictionary to fetch.
+ *
+ * Returns: A pointer to a bytea containing the dictionary data.
+ */
+bytea *
+get_zstd_dict(Oid dictid)
+{
+ HeapTuple tuple;
+ Datum datum;
+ bool isNull;
+ bytea *dict_bytea;
+ Size bytea_len;
+ bytea *result;
+
+ tuple = SearchSysCache1(ZSTDDICTIDOID, ObjectIdGetDatum(dictid));
+ if (!HeapTupleIsValid(tuple))
+ ereport(ERROR, (errmsg("Cache lookup failed for dictid %u", dictid)));
+
+ datum = SysCacheGetAttr(ZSTDDICTIDOID, tuple, Anum_pg_zstd_dictionaries_dict, &isNull);
+ if (isNull)
+ ereport(ERROR, (errmsg("Dictionary not found for dictid %u", dictid)));
+
+ dict_bytea = DatumGetByteaP(datum);
+ bytea_len = VARSIZE(dict_bytea);
+
+ result = palloc(bytea_len);
+ memcpy(result, dict_bytea, bytea_len);
+
+ ReleaseSysCache(tuple);
+
+ return result;
+}
diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c
index ceb9a229b63..58bc92d5959 100644
--- a/src/backend/commands/dropcmds.c
+++ b/src/backend/commands/dropcmds.c
@@ -508,6 +508,7 @@ does_not_exist_skipping(ObjectType objtype, Node *object)
case OBJECT_PUBLICATION_REL:
case OBJECT_TABCONSTRAINT:
case OBJECT_USER_MAPPING:
+ case OBJECT_ZSTD_DICTIONARY:
/* These are currently not used or needed. */
elog(ERROR, "unsupported object type: %d", (int) objtype);
break;
diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c
index edc2c988e29..c5c9291c384 100644
--- a/src/backend/commands/event_trigger.c
+++ b/src/backend/commands/event_trigger.c
@@ -2186,6 +2186,7 @@ stringify_grant_objtype(ObjectType objtype)
case OBJECT_TSTEMPLATE:
case OBJECT_USER_MAPPING:
case OBJECT_VIEW:
+ case OBJECT_ZSTD_DICTIONARY:
elog(ERROR, "unsupported object type: %d", (int) objtype);
}
@@ -2270,6 +2271,7 @@ stringify_adefprivs_objtype(ObjectType objtype)
case OBJECT_TSTEMPLATE:
case OBJECT_USER_MAPPING:
case OBJECT_VIEW:
+ case OBJECT_ZSTD_DICTIONARY:
elog(ERROR, "unsupported object type: %d", (int) objtype);
}
diff --git a/src/backend/commands/seclabel.c b/src/backend/commands/seclabel.c
index cee5d7bbb9c..e4fe6a64b2e 100644
--- a/src/backend/commands/seclabel.c
+++ b/src/backend/commands/seclabel.c
@@ -92,6 +92,7 @@ SecLabelSupportsObjectType(ObjectType objtype)
case OBJECT_TSPARSER:
case OBJECT_TSTEMPLATE:
case OBJECT_USER_MAPPING:
+ case OBJECT_ZSTD_DICTIONARY:
return false;
/*
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 3e4d5568bde..063780e56dc 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5301,6 +5301,9 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_ZSTD_COMPRESSION_ID:
+ result = "zstd";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 60b12446a1c..b4b0e44d7ea 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -460,6 +460,9 @@ static const struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd", TOAST_ZSTD_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 34826d01380..4dd6da32324 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -756,7 +756,7 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate
#row_security = on
#default_table_access_method = 'heap'
#default_tablespace = '' # a tablespace name, '' uses the default
-#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
+#default_toast_compression = 'pglz' # 'pglz' or 'lz4' or 'zstd'
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#check_function_bodies = on
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 1d08268393e..26951f8f890 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -2171,8 +2171,9 @@ describeOneTableDetails(const char *schemaname,
/* these strings are literal in our syntax, so not translated. */
printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
(compression[0] == 'l' ? "lz4" :
- (compression[0] == '\0' ? "" :
- "???"))),
+ (compression[0] == 'z' ? "zstd" :
+ (compression[0] == '\0' ? "" :
+ "???")))),
false, false);
}
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index c916b9299a8..f507f7111c4 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -2875,11 +2875,11 @@ match_previous_words(int pattern_id,
/* ALTER TABLE ALTER [COLUMN] <foo> SET ( */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "("))
- COMPLETE_WITH("n_distinct", "n_distinct_inherited");
+ COMPLETE_WITH("n_distinct", "n_distinct_inherited", "zstd_level", "zstd_dict_size");
/* ALTER TABLE ALTER [COLUMN] <foo> SET COMPRESSION */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION"))
- COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4");
+ COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD");
/* ALTER TABLE ALTER [COLUMN] <foo> SET EXPRESSION */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "EXPRESSION") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "EXPRESSION"))
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 13c4612ceed..fe4fc3db471 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -13,6 +13,10 @@
#ifndef TOAST_COMPRESSION_H
#define TOAST_COMPRESSION_H
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
+
/*
* GUC support.
*
@@ -38,7 +42,8 @@ typedef enum ToastCompressionId
{
TOAST_PGLZ_COMPRESSION_ID = 0,
TOAST_LZ4_COMPRESSION_ID = 1,
- TOAST_INVALID_COMPRESSION_ID = 2,
+ TOAST_ZSTD_COMPRESSION_ID = 2,
+ TOAST_INVALID_COMPRESSION_ID = 3
} ToastCompressionId;
/*
@@ -48,10 +53,24 @@ typedef enum ToastCompressionId
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_COMPRESSION 'z'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
+#define InvalidDictId 0
+
+#ifdef USE_ZSTD
+#define DEFAULT_ZSTD_LEVEL ZSTD_CLEVEL_DEFAULT
+#define MIN_ZSTD_LEVEL (int)-ZSTD_BLOCKSIZE_MAX
+#define MAX_ZSTD_LEVEL 22
+#define DEFAULT_ZSTD_DICT_SIZE (4 * 1024) /* 4 KB */
+#else
+#define DEFAULT_ZSTD_LEVEL 0
+#define MIN_ZSTD_LEVEL 0
+#define MAX_ZSTD_LEVEL 0
+#define DEFAULT_ZSTD_DICT_SIZE 0
+#endif
/* pglz compression/decompression routines */
extern struct varlena *pglz_compress_datum(const struct varlena *value);
@@ -65,6 +84,11 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* zstd compression/decompression routines */
+extern struct varlena *zstd_compress_datum(const struct varlena *value, Oid dictid, int zstd_level);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index e6ab8afffb6..08bf3dfc673 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -33,6 +33,8 @@ typedef struct
int32 tai_size;
uint8 tai_colflags;
char tai_compression;
+ Oid dictid;
+ int zstd_level;
} ToastAttrInfo;
/*
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index 06ae8583c1e..395afe915d8 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -27,25 +27,54 @@ typedef struct toast_compress_header
* external size; see va_extinfo */
} toast_compress_header;
+typedef struct toast_compress_header_ext
+{
+ int32 vl_len_; /* varlena header (do not touch directly!) */
+ uint32 tcinfo; /* 2 bits for compression method and 30 bits
+ * external size; see va_extinfo */
+ uint32 ext_alg; /* compression method */
+ Oid dictid; /* Dictionary Id */
+} toast_compress_header_ext;
+
+typedef struct CompressionInfo
+{
+ char cmethod;
+ Oid dictid;
+ int zstd_level; /* ZSTD compression level */
+} CompressionInfo;
+
/*
* Utilities for manipulation of header information for compressed
* toast entries.
*/
#define TOAST_COMPRESS_EXTSIZE(ptr) \
(((toast_compress_header *) (ptr))->tcinfo & VARLENA_EXTSIZE_MASK)
-#define TOAST_COMPRESS_METHOD(ptr) \
- (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
+#define TOAST_COMPRESS_METHOD(PTR) \
+ ( ((((toast_compress_header *) (PTR))->tcinfo >> VARLENA_EXTSIZE_BITS) == VARLENA_EXTENDED_COMPRESSION_FLAG ) \
+ ? (((toast_compress_header_ext *) (PTR))->ext_alg) \
+ : ( (((toast_compress_header *) (PTR))->tcinfo) >> VARLENA_EXTSIZE_BITS ) )
-#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
+#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method, dictid) \
do { \
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_compress_header *) (ptr))->tcinfo = \
- (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_ZSTD_COMPRESSION_ID); \
+ /* If the compression method is less than TOAST_ZSTD_COMPRESSION_ID, don't use ext_alg */ \
+ if ((cm_method) < TOAST_ZSTD_COMPRESSION_ID) { \
+ ((toast_compress_header *) (ptr))->tcinfo = \
+ (len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+ } else { \
+ /* For compression methods after lz4, use 'VARLENA_EXTENDED_COMPRESSION_FLAG' \
+ in the top bits of tcinfo to indicate compression algorithm is stored in ext_alg. */ \
+ ((toast_compress_header_ext *) (ptr))->tcinfo = \
+ (len) | ((uint32)VARLENA_EXTENDED_COMPRESSION_FLAG << VARLENA_EXTSIZE_BITS); \
+ ((toast_compress_header_ext *) (ptr))->ext_alg = (cm_method); \
+ ((toast_compress_header_ext *) (ptr))->dictid = (dictid); \
+ } \
} while (0)
-extern Datum toast_compress_datum(Datum value, char cmethod);
+extern Datum toast_compress_datum(Datum value, CompressionInfo cmp);
extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
@@ -59,5 +88,6 @@ extern int toast_open_indexes(Relation toastrel,
extern void toast_close_indexes(Relation *toastidxs, int num_indexes,
LOCKMODE lock);
extern Snapshot get_toast_snapshot(void);
+extern void setup_compression_info(CompressionInfo * info, Form_pg_attribute att);
#endif /* TOAST_INTERNALS_H */
diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile
index 2bbc7805fe3..1ecd76dd312 100644
--- a/src/include/catalog/Makefile
+++ b/src/include/catalog/Makefile
@@ -81,7 +81,8 @@ CATALOG_HEADERS := \
pg_publication_namespace.h \
pg_publication_rel.h \
pg_subscription.h \
- pg_subscription_rel.h
+ pg_subscription_rel.h \
+ pg_zstd_dictionaries.h
GENERATED_HEADERS := $(CATALOG_HEADERS:%.h=%_d.h)
diff --git a/src/include/catalog/meson.build b/src/include/catalog/meson.build
index ec1cf467f6f..e9cb6d911cc 100644
--- a/src/include/catalog/meson.build
+++ b/src/include/catalog/meson.build
@@ -69,6 +69,7 @@ catalog_headers = [
'pg_publication_rel.h',
'pg_subscription.h',
'pg_subscription_rel.h',
+ 'pg_zstd_dictionaries.h',
]
# The .dat files we need can just be listed alphabetically.
diff --git a/src/include/catalog/pg_zstd_dictionaries.h b/src/include/catalog/pg_zstd_dictionaries.h
new file mode 100644
index 00000000000..cf847ee2801
--- /dev/null
+++ b/src/include/catalog/pg_zstd_dictionaries.h
@@ -0,0 +1,48 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_zstd_dictionaries.h
+ * definition of the "zstd dictionary" system catalog (pg_zstd_dictionaries)
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ *
+ * src/include/catalog/pg_zstd_dictionaries.h
+ *
+ * NOTES
+ * The Catalog.pm module reads this file and derives schema
+ * information.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_ZSTD_DICTIONARIES_H
+#define PG_ZSTD_DICTIONARIES_H
+
+#include "catalog/genbki.h"
+#include "catalog/pg_zstd_dictionaries_d.h"
+
+/* ----------------
+ * pg_zstd_dictionaries definition. cpp turns this into
+ * typedef struct FormData_pg_zstd_dictionaries
+ * ----------------
+ */
+CATALOG(pg_zstd_dictionaries,9946,ZstdDictionariesRelationId)
+{
+ Oid dictid;
+
+ /*
+ * variable-length fields start here, but we allow direct access to dict
+ */
+ bytea dict BKI_FORCE_NOT_NULL;
+} FormData_pg_zstd_dictionaries;
+
+/* Pointer type to a tuple with the format of pg_zstd_dictionaries relation */
+typedef FormData_pg_zstd_dictionaries *Form_pg_zstd_dictionaries;
+
+DECLARE_TOAST_WITH_MACRO(pg_zstd_dictionaries, 9947, 9948, PgZstdDictionariesToastTable, PgZstdDictionariesToastIndex);
+
+DECLARE_UNIQUE_INDEX_PKEY(pg_zstd_dictionaries_dictid_index, 9949, ZstdDictidIndexId, pg_zstd_dictionaries, btree(dictid oid_ops));
+
+MAKE_SYSCACHE(ZSTDDICTIDOID, pg_zstd_dictionaries_dictid_index, 128);
+
+extern bytea *get_zstd_dict(Oid dictid);
+
+#endif /* PG_ZSTD_DICTIONARIES_H */
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 4610fc61293..a74c93c020c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -2366,6 +2366,7 @@ typedef enum ObjectType
OBJECT_TYPE,
OBJECT_USER_MAPPING,
OBJECT_VIEW,
+ OBJECT_ZSTD_DICTIONARY,
} ObjectType;
/* ----------------------
diff --git a/src/include/utils/attoptcache.h b/src/include/utils/attoptcache.h
index f684a772af5..ee16bf3a4d4 100644
--- a/src/include/utils/attoptcache.h
+++ b/src/include/utils/attoptcache.h
@@ -21,6 +21,12 @@ typedef struct AttributeOpts
int32 vl_len_; /* varlena header (do not touch directly!) */
float8 n_distinct;
float8 n_distinct_inherited;
+ float8 dictid; /* Oid is a 32-bit unsigned integer, but
+ * relopt_int is limited to INT_MAX, so it
+ * cannot represent the full range of Oid
+ * values. */
+ int zstd_dict_size;
+ int zstd_level;
} AttributeOpts;
extern AttributeOpts *get_attribute_options(Oid attrelid, int attnum);
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 4dd9ee7200d..94495388ade 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -238,10 +238,11 @@ NOTICE: merging multiple inherited definitions of column "f1"
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd.
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
-- test alter compression method
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 7bd7642b4b9..0ce49152176 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -233,6 +233,9 @@ HINT: Available values: pglz.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
HINT: Available values: pglz.
+SET default_toast_compression = 'zstd';
+ERROR: invalid value for parameter "default_toast_compression": "zstd"
+HINT: Available values: pglz.
SET default_toast_compression = 'lz4';
ERROR: invalid value for parameter "default_toast_compression": "lz4"
HINT: Available values: pglz.
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index 490595fcfb2..e29909558f9 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -102,6 +102,7 @@ CREATE TABLE cminh() INHERITS (cmdata, cmdata3);
-- test default_toast_compression GUC
SET default_toast_compression = '';
SET default_toast_compression = 'I do not exist compression';
+SET default_toast_compression = 'zstd';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d16bc208654..86fb79b2076 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -905,6 +905,7 @@ FormData_pg_ts_parser
FormData_pg_ts_template
FormData_pg_type
FormData_pg_user_mapping
+FormData_pg_zstd_dictionaries
FormExtraData_pg_attribute
Form_pg_aggregate
Form_pg_am
@@ -964,6 +965,7 @@ Form_pg_ts_parser
Form_pg_ts_template
Form_pg_type
Form_pg_user_mapping
+Form_pg_zstd_dictionaries
FormatNode
FreeBlockNumberArray
FreeListData
--
2.47.1
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-18 16:22 Robert Haas <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
0 siblings, 3 replies; 21+ messages in thread
From: Robert Haas @ 2025-04-18 16:22 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: pgsql-hackers
On Tue, Apr 15, 2025 at 2:13 PM Nikhil Kumar Veldanda
<[email protected]> wrote:
> Addressing Compressed Datum Leaks problem (via CTAS, INSERT INTO ... SELECT ...)
>
> As compressed datums can be copied to other unrelated tables via CTAS,
> INSERT INTO ... SELECT, or CREATE TABLE ... EXECUTE, I’ve introduced a
> method inheritZstdDictionaryDependencies. This method is invoked at
> the end of such statements and ensures that any dictionary
> dependencies from source tables are copied to the destination table.
> We determine the set of source tables using the relationOids field in
> PlannedStmt.
With the disclaimer that I haven't opened the patch or thought
terribly deeply about this issue, at least not yet, my fairly strong
suspicion is that this design is not going to work out, for multiple
reasons. In no particular order:
1. I don't think users will like it if dependencies on a zstd
dictionary spread like kudzu across all of their tables. I don't think
they'd like it even if it were 100% accurate, but presumably this is
going to add dependencies any time there MIGHT be a real dependency
rather than only when there actually is one.
2. Inserting into a table or updating it only takes RowExclusiveLock,
which is not even self-exclusive. I doubt that it's possible to change
system catalogs in a concurrency-safe way with such a weak lock. For
instance, if two sessions tried to do the same thing in concurrent
transactions, they could both try to add the same dependency at the
same time.
3. I'm not sure that CTAS, INSERT INTO...SELECT, and CREATE
TABLE...EXECUTE are the only ways that datums can creep from one table
into another. For example, what if I create a plpgsql function that
gets a value from one table and stores it in a variable, and then use
that variable to drive an INSERT into another table? I seem to recall
there are complex cases involving records and range types and arrays,
too, where the compressed object gets wrapped inside of another
object; though maybe that wouldn't matter to your implementation if
INSERT INTO ... SELECT uses a sufficiently aggressive strategy for
adding dependencies.
When Dilip and I were working on lz4 TOAST compression, my first
instinct was to not let LZ4-compressed datums leak out of a table by
forcing them to be decompressed (and then possibly recompressed). We
spent a long time trying to make that work before giving up. I think
this is approximately where things started to unravel, and I'd suggest
you read both this message and some of the discussion before and
after:
https://www.postgresql.org/message-id/[email protected]
I think we could add plain-old zstd compression without really
tackling this issue, but if we are going to add dictionaries then I
think we might need to revisit the idea of preventing things from
leaking out of tables. What I can't quite remember at the moment is
how much of the problem was that it was going to be slow to force the
recompression, and how much of it was that we weren't sure we could
even find all the places in the code that might need such handling.
I'm now also curious to know whether Andres would agree that it's bad
if zstd dictionaries are un-droppable. After all, I thought it would
be bad if there was no way to eliminate a dependency on a compression
method, and he disagreed. So maybe he would also think undroppable
dictionaries are fine. But maybe not. It seems even worse to me than
undroppable compression methods, because you'll probably not have that
many compression methods ever, but you could have a large number of
dictionaries eventually.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-21 07:02 Michael Paquier <[email protected]>
parent: Robert Haas <[email protected]>
2 siblings, 0 replies; 21+ messages in thread
From: Michael Paquier @ 2025-04-21 07:02 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nikhil Kumar Veldanda <[email protected]>; pgsql-hackers
On Fri, Apr 18, 2025 at 12:22:18PM -0400, Robert Haas wrote:
> I think we could add plain-old zstd compression without really
> tackling this issue, but if we are going to add dictionaries then I
> think we might need to revisit the idea of preventing things from
> leaking out of tables. What I can't quite remember at the moment is
> how much of the problem was that it was going to be slow to force the
> recompression, and how much of it was that we weren't sure we could
> even find all the places in the code that might need such handling.
FWIW, this point resonates here. There is one thing that we have to
do anyway: we just have one bit left in the varlena headers as lz4 is
using the one before last. So we have to make it extensible, even if
it means that any compression method other than LZ4 and pglz would
consume one more byte in its header by default. And I think that this
has to happen at some point if we want flexibility in this area.
+ struct
+ {
+ uint32 va_header;
+ uint32 va_tcinfo;
+ uint32 va_cmp_alg;
+ uint32 va_cmp_dictid;
+ char va_data[FLEXIBLE_ARRAY_MEMBER];
+ } va_compressed_ext;
Speaking of which, I am confused by this abstraction choice in
varatt.h in the first patch. Are we sure that we are always going to
have a dictionary attached to a compressed data set or even a
va_cmp_alg? It seems to me that this could lead to a waste of data in
some cases because these fields may not be required depending on the
compression method used, as some fields may not care about these
details. This kind of data should be made optional, on a per-field
basis.
One thing that I've been wondering is how it would be possible to make
the area around varattrib_4b more readable while dealing with more
extensibility. It would be a good occasion to improve that, even if
I'm hand-waving here currently and that the majority of this code is
old enough to vote, with few modifications across the years.
The second thing that I'd love to see on top of the addition of the
extensibility is adding plain compression support for zstd, with
nothing fancy, just the compression and decompression bits. I've done
quite a few benchmarks with the two, and results kind of point in the
direction that zstd is more efficient than lz4 overall. Don't take me
wrong: lz4 can be better in some workloads as it can consume less CPU
than zstd while compressing less. However, a comparison of ratios
like (compression rate / cpu used) has always led me to see zstd as
superior in a large number of cases. lz4 is still very good if you
are CPU-bound and don't care about the extra space required. Both are
three classes better than pglz.
Once we have these three points incrementally built-in together (the
last bit extensibility, the potential varatt.h refactoring and the
zstd support), there may be a point in having support for more
advanced options with the compression methods in the shape of dicts or
more requirements linked to other compression methods, but I think the
topic is complex enough that we should make sure that these basics are
implemented in a way sane enough so as we'd be able to extend them
with all the use cases in mind.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-22 00:52 Nikhil Kumar Veldanda <[email protected]>
parent: Robert Haas <[email protected]>
2 siblings, 1 reply; 21+ messages in thread
From: Nikhil Kumar Veldanda @ 2025-04-22 00:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: pgsql-hackers
Hi Robert,
Thank you for your feedback on the patch. You’re right that my
proposed design will introduce more dictionary dependencies as
dictionaries grow, I chose this path specifically to avoid changing
existing system behavior and prevent perf regressions in CTAS and
related commands.
After reviewing the email thread you attached on previous response, I
identified a natural choke point for both inserts and updates: the
call to "heap_toast_insert_or_update" inside
heap_prepare_insert/heap_update. In the current master branch, that
function only runs when HeapTupleHasExternal is true; my patch extends
it to HeapTupleHasVarWidth tuples as well. By decompressing every
nested compressed datum at this point—no matter how deeply nested—we
can prevent any leaked datum from propagating into unrelated tables.
This mirrors the existing inlining logic in toast_tuple_init for
external toasted datum, but takes it one step further to fully flatten
datum(decompress datum, not just top level at every level).
On the performance side, my basic benchmarks show almost no regression
for simple INSERT … VALUES workloads. CTAS, however, does regress
noticeably: a CTAS completes in about 4 seconds before this patch, but
with this patch it takes roughly 24 seconds. (For reference, a normal
insert into the source table took about 58 seconds when using zstd
dictionary compression), I suspect the extra cost comes from the added
zstd decompression and PGLZ compression on the destination table.
I’ve attached v13-0008-initial-draft-to-address-datum-leak-problem.patch,
which implements this “flatten_datum” method.
I’d love to know your thoughts on this. Am I on the right track for
solving the problem?
Best regards,
Nikhil Veldanda
On Fri, Apr 18, 2025 at 9:22 AM Robert Haas <[email protected]> wrote:
>
> On Tue, Apr 15, 2025 at 2:13 PM Nikhil Kumar Veldanda
> <[email protected]> wrote:
> > Addressing Compressed Datum Leaks problem (via CTAS, INSERT INTO ... SELECT ...)
> >
> > As compressed datums can be copied to other unrelated tables via CTAS,
> > INSERT INTO ... SELECT, or CREATE TABLE ... EXECUTE, I’ve introduced a
> > method inheritZstdDictionaryDependencies. This method is invoked at
> > the end of such statements and ensures that any dictionary
> > dependencies from source tables are copied to the destination table.
> > We determine the set of source tables using the relationOids field in
> > PlannedStmt.
>
> With the disclaimer that I haven't opened the patch or thought
> terribly deeply about this issue, at least not yet, my fairly strong
> suspicion is that this design is not going to work out, for multiple
> reasons. In no particular order:
>
> 1. I don't think users will like it if dependencies on a zstd
> dictionary spread like kudzu across all of their tables. I don't think
> they'd like it even if it were 100% accurate, but presumably this is
> going to add dependencies any time there MIGHT be a real dependency
> rather than only when there actually is one.
>
> 2. Inserting into a table or updating it only takes RowExclusiveLock,
> which is not even self-exclusive. I doubt that it's possible to change
> system catalogs in a concurrency-safe way with such a weak lock. For
> instance, if two sessions tried to do the same thing in concurrent
> transactions, they could both try to add the same dependency at the
> same time.
>
> 3. I'm not sure that CTAS, INSERT INTO...SELECT, and CREATE
> TABLE...EXECUTE are the only ways that datums can creep from one table
> into another. For example, what if I create a plpgsql function that
> gets a value from one table and stores it in a variable, and then use
> that variable to drive an INSERT into another table? I seem to recall
> there are complex cases involving records and range types and arrays,
> too, where the compressed object gets wrapped inside of another
> object; though maybe that wouldn't matter to your implementation if
> INSERT INTO ... SELECT uses a sufficiently aggressive strategy for
> adding dependencies.
>
> When Dilip and I were working on lz4 TOAST compression, my first
> instinct was to not let LZ4-compressed datums leak out of a table by
> forcing them to be decompressed (and then possibly recompressed). We
> spent a long time trying to make that work before giving up. I think
> this is approximately where things started to unravel, and I'd suggest
> you read both this message and some of the discussion before and
> after:
>
> https://www.postgresql.org/message-id/[email protected]
>
> I think we could add plain-old zstd compression without really
> tackling this issue, but if we are going to add dictionaries then I
> think we might need to revisit the idea of preventing things from
> leaking out of tables. What I can't quite remember at the moment is
> how much of the problem was that it was going to be slow to force the
> recompression, and how much of it was that we weren't sure we could
> even find all the places in the code that might need such handling.
>
> I'm now also curious to know whether Andres would agree that it's bad
> if zstd dictionaries are un-droppable. After all, I thought it would
> be bad if there was no way to eliminate a dependency on a compression
> method, and he disagreed. So maybe he would also think undroppable
> dictionaries are fine. But maybe not. It seems even worse to me than
> undroppable compression methods, because you'll probably not have that
> many compression methods ever, but you could have a large number of
> dictionaries eventually.
>
> --
> Robert Haas
> EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v13-0008-initial-draft-to-address-datum-leak-problem.patch (10.6K, ../../CAFAfj_Fee7zQ7YNnJObLMqVF=MpFwVHszzPJ-ZWkiiade04SAw@mail.gmail.com/2-v13-0008-initial-draft-to-address-datum-leak-problem.patch)
download | inline diff:
From 79d501b31c967e2e01424ba10cd0d0e14d175c08 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <[email protected]>
Date: Mon, 21 Apr 2025 13:12:30 +0000
Subject: [PATCH v13 8/8] initial draft to address datum leak problem
---
src/backend/access/heap/heapam.c | 3 +-
src/backend/access/table/toast_helper.c | 227 +++++++++++++++++-
src/backend/catalog/pg_zstd_dictionaries.c | 4 +-
src/test/regress/expected/compression.out | 8 +-
.../regress/expected/compression_zstd_1.out | 60 ++---
5 files changed, 264 insertions(+), 38 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index c1a4de14a59..0348161432a 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2278,7 +2278,7 @@ heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
Assert(!HeapTupleHasExternal(tup));
return tup;
}
- else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
+ else if (HeapTupleHasExternal(tup) || HeapTupleHasVarWidth(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
return heap_toast_insert_or_update(relation, tup, NULL, options);
else
return tup;
@@ -3776,6 +3776,7 @@ l2:
else
need_toast = (HeapTupleHasExternal(&oldtup) ||
HeapTupleHasExternal(newtup) ||
+ HeapTupleHasVarWidth(newtup) ||
newtup->t_len > TOAST_TUPLE_THRESHOLD);
pagefree = PageGetHeapFreeSpace(page);
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index f4b1cbe494e..2a90ebf77c9 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -19,7 +19,17 @@
#include "access/toast_internals.h"
#include "catalog/pg_type_d.h"
#include "varatt.h"
-
+#include "utils/lsyscache.h"
+#include "access/htup_details.h"
+#include "catalog/pg_type.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/rangetypes.h"
+#include "utils/multirangetypes.h"
+#include "utils/typcache.h"
+#include "miscadmin.h"
+
+static Datum flatten_datum(Datum value, Oid typid);
/*
* Prepare to TOAST a tuple.
@@ -151,6 +161,25 @@ toast_tuple_init(ToastTupleContext *ttc)
ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
}
+ if (!(IsCatalogNamespace(ttc->ttc_rel->rd_rel->relnamespace) || IsToastNamespace(ttc->ttc_rel->rd_rel->relnamespace)))
+ {
+ if (!VARATT_IS_SHORT(new_value))
+ {
+ Datum oldd = PointerGetDatum(new_value);
+ Datum clean = flatten_datum(oldd, att->atttypid);
+
+ if (DatumGetPointer(clean) != DatumGetPointer(oldd))
+ {
+ if (ttc->ttc_attr[i].tai_oldexternal != NULL)
+ pfree(new_value);
+ new_value = (struct varlena *) DatumGetPointer(clean);
+ ttc->ttc_values[i] = clean;
+ ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
+ ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+ }
+ }
+ }
+
/*
* Remember the size of this attribute
*/
@@ -346,3 +375,199 @@ toast_delete_external(Relation rel, const Datum *values, const bool *isnull,
}
}
}
+
+static Datum
+flatten_datum(Datum value, Oid typid)
+{
+ Oid basetypid;
+ char typtype;
+
+ /* initialize at top of this block */
+ basetypid = getBaseType(typid); /* Get basetype for Domain. */
+ typtype = get_typtype(basetypid);
+
+ /* ---------- BASE / ENUM / PSEUDO ----------------------- */
+ if (typtype == TYPTYPE_BASE ||
+ typtype == TYPTYPE_ENUM ||
+ typtype == TYPTYPE_PSEUDO)
+ {
+ if (get_typlen(basetypid) > 0 ||
+ get_typbyval(basetypid))
+ return value; /* fixed‑len or pass‑by‑val */
+
+ /* ---------- ARRAY ------------------------------------------------ */
+ if (type_is_array(typid))
+ {
+ ArrayType *arr;
+ TypeCacheEntry *elemCache;
+ Datum *elems;
+ bool *nulls;
+ int nitems;
+ int i;
+ ArrayType *new_arr;
+
+ arr = DatumGetArrayTypeP(value);
+ elemCache = lookup_type_cache(ARR_ELEMTYPE(arr), 0);
+
+ if (elemCache->typbyval || elemCache->typlen > 0)
+ return PointerGetDatum(arr);
+
+ deconstruct_array(arr,
+ ARR_ELEMTYPE(arr),
+ elemCache->typlen,
+ elemCache->typbyval,
+ elemCache->typalign,
+ &elems, &nulls, &nitems);
+
+ if (nitems == 0)
+ return PointerGetDatum(arr);
+
+ for (i = 0; i < nitems; i++)
+ {
+ if (!nulls[i])
+ elems[i] = flatten_datum(elems[i],
+ ARR_ELEMTYPE(arr));
+ }
+
+ new_arr = construct_md_array(elems, nulls,
+ ARR_NDIM(arr),
+ ARR_DIMS(arr),
+ ARR_LBOUND(arr),
+ ARR_ELEMTYPE(arr),
+ elemCache->typlen,
+ elemCache->typbyval,
+ elemCache->typalign);
+
+ pfree(elems);
+ pfree(nulls);
+
+ return PointerGetDatum(new_arr);
+ }
+
+ return PointerGetDatum(PG_DETOAST_DATUM(value));
+ }
+
+ /* ---------- COMPOSITE ------------------------------------------- */
+ if (typtype == TYPTYPE_COMPOSITE)
+ {
+ HeapTupleHeader hdr;
+ TupleDesc td;
+ int natts;
+ Datum *vals;
+ bool *nulls;
+ HeapTupleData t;
+ int i;
+ Form_pg_attribute att;
+ HeapTuple newt;
+ HeapTupleHeader copy;
+
+ hdr = DatumGetHeapTupleHeader(value);
+
+ if (!(hdr->t_infomask & HEAP_HASVARWIDTH))
+ return PointerGetDatum(hdr);
+
+ td = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(hdr),
+ HeapTupleHeaderGetTypMod(hdr));
+ natts = td->natts;
+
+ vals = palloc(sizeof(Datum) * natts);
+ nulls = palloc(sizeof(bool) * natts);
+
+ t.t_len = VARSIZE(hdr); /* ⚑ portable */
+ t.t_tableOid = InvalidOid;
+ t.t_data = hdr;
+ ItemPointerSetInvalid(&t.t_self);
+
+ heap_deform_tuple(&t, td, vals, nulls);
+
+ for (i = 0; i < natts; i++)
+ {
+ att = TupleDescAttr(td, i);
+ if (att->attisdropped || att->atthasmissing)
+ continue;
+ if (!nulls[i] && att->attlen == -1)
+ vals[i] = flatten_datum(vals[i], att->atttypid);
+ }
+
+ newt = heap_form_tuple(td, vals, nulls);
+ copy = (HeapTupleHeader) palloc(newt->t_len);
+ memcpy(copy, newt->t_data, newt->t_len);
+
+ ReleaseTupleDesc(td);
+ pfree(vals);
+ pfree(nulls);
+ heap_freetuple(newt);
+
+ return PointerGetDatum(copy);
+ }
+
+ /* ---------- RANGE ----------------------------------------------- */
+ if (typtype == TYPTYPE_RANGE)
+ {
+ RangeType *r;
+ TypeCacheEntry *tc;
+ RangeBound l;
+ RangeBound u;
+ bool empty;
+
+ r = DatumGetRangeTypeP(value);
+ tc = lookup_type_cache(basetypid, TYPECACHE_RANGE_INFO);
+
+ range_deserialize(tc, r, &l, &u, &empty);
+
+ if (!empty &&
+ !(tc->rngelemtype->typbyval ||
+ tc->rngelemtype->typlen > 0))
+ {
+ if (!l.infinite)
+ l.val = flatten_datum(l.val,
+ tc->rngelemtype->type_id);
+ if (!u.infinite)
+ u.val = flatten_datum(u.val,
+ tc->rngelemtype->type_id);
+ return PointerGetDatum(make_range(tc, &l, &u, empty, NULL));
+ }
+
+ return PointerGetDatum(r);
+ }
+
+ /* ---------- MULTIRANGE ---------------------------- */
+ if (typtype == TYPTYPE_MULTIRANGE)
+ {
+ MultirangeType *mr;
+ TypeCacheEntry *tc;
+ int32 rangeCount;
+ RangeType **ranges;
+ RangeType **out;
+ MultirangeType *newmr;
+ int i;
+
+ mr = DatumGetMultirangeTypeP(value);
+ tc = lookup_type_cache(basetypid,
+ TYPECACHE_MULTIRANGE_INFO);
+
+ multirange_deserialize(tc->rngtype, mr,
+ &rangeCount, &ranges);
+
+ out = palloc(sizeof(RangeType *) * rangeCount);
+
+ for (i = 0; i < rangeCount; i++)
+ {
+ out[i] = DatumGetRangeTypeP(
+ flatten_datum(PointerGetDatum(ranges[i]),
+ RangeTypeGetOid(ranges[i])));
+ }
+
+ newmr = make_multirange(MultirangeTypeGetOid(mr),
+ tc->rngtype,
+ rangeCount, out);
+ pfree(out);
+
+ return PointerGetDatum(newmr);
+ }
+
+ elog(ERROR, "flatten_datum: unsupported type %u", typid);
+ pg_unreachable();
+ /* keep compiler happy: */
+ return (Datum) 0;
+}
diff --git a/src/backend/catalog/pg_zstd_dictionaries.c b/src/backend/catalog/pg_zstd_dictionaries.c
index 08a6883ecd4..63c2a34190a 100644
--- a/src/backend/catalog/pg_zstd_dictionaries.c
+++ b/src/backend/catalog/pg_zstd_dictionaries.c
@@ -410,7 +410,7 @@ range_typzstdsampling(PG_FUNCTION_ARGS)
bool empty;
/* Get information about range type; note column might be a domain */
- TypeCacheEntry *typcache = range_get_typcache(fcinfo, getBaseType(range->rangetypid));
+ TypeCacheEntry *typcache = range_get_typcache(fcinfo, RangeTypeGetOid(range));
/* If the type does not supply a builder, skip */
if (!OidIsValid(typcache->rngelemtype->typzstdsampling))
@@ -436,7 +436,7 @@ multirange_typzstdsampling(PG_FUNCTION_ARGS)
RangeType **ranges;
/* Get information about multirange type; note column might be a domain */
- TypeCacheEntry *typcache = multirange_get_typcache(fcinfo, getBaseType(mrange->multirangetypid));
+ TypeCacheEntry *typcache = multirange_get_typcache(fcinfo, MultirangeTypeGetOid(mrange));
/* If the type does not supply a builder, skip */
if (!OidIsValid(typcache->rngtype->typzstdsampling))
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 94495388ade..29ef885e516 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -69,7 +69,7 @@ SELECT pg_column_compression(f1) FROM cmmove3;
pg_column_compression
-----------------------
pglz
- lz4
+ pglz
(2 rows)
-- test LIKE INCLUDING COMPRESSION
@@ -97,7 +97,7 @@ UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
SELECT pg_column_compression(f1) FROM cmmove2;
pg_column_compression
-----------------------
- lz4
+ pglz
(1 row)
-- test externally stored compressed data
@@ -200,8 +200,8 @@ SELECT pg_column_compression(f1) FROM cmdata1;
SELECT pg_column_compression(x) FROM compressmv;
pg_column_compression
-----------------------
- lz4
- lz4
+ pglz
+ pglz
(2 rows)
-- test compression with partition
diff --git a/src/test/regress/expected/compression_zstd_1.out b/src/test/regress/expected/compression_zstd_1.out
index c1a7936e574..260b9fd2b17 100644
--- a/src/test/regress/expected/compression_zstd_1.out
+++ b/src/test/regress/expected/compression_zstd_1.out
@@ -139,36 +139,36 @@ SELECT pg_column_compression(f1) AS mv_compression
FROM compressmv_zstd;
mv_compression
----------------
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
- zstd
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
+ pglz
(30 rows)
SELECT objid::regclass, refobjid from pg_depend where refclassid = 9946;
--
2.47.1
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-22 16:24 Andres Freund <[email protected]>
parent: Robert Haas <[email protected]>
2 siblings, 0 replies; 21+ messages in thread
From: Andres Freund @ 2025-04-22 16:24 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nikhil Kumar Veldanda <[email protected]>; pgsql-hackers
Hi,
On 2025-04-18 12:22:18 -0400, Robert Haas wrote:
> On Tue, Apr 15, 2025 at 2:13 PM Nikhil Kumar Veldanda
> <[email protected]> wrote:
> > Addressing Compressed Datum Leaks problem (via CTAS, INSERT INTO ... SELECT ...)
> >
> > As compressed datums can be copied to other unrelated tables via CTAS,
> > INSERT INTO ... SELECT, or CREATE TABLE ... EXECUTE, I’ve introduced a
> > method inheritZstdDictionaryDependencies. This method is invoked at
> > the end of such statements and ensures that any dictionary
> > dependencies from source tables are copied to the destination table.
> > We determine the set of source tables using the relationOids field in
> > PlannedStmt.
>
> With the disclaimer that I haven't opened the patch or thought
> terribly deeply about this issue, at least not yet, my fairly strong
> suspicion is that this design is not going to work out, for multiple
> reasons. In no particular order:
>
> 1. I don't think users will like it if dependencies on a zstd
> dictionary spread like kudzu across all of their tables. I don't think
> they'd like it even if it were 100% accurate, but presumably this is
> going to add dependencies any time there MIGHT be a real dependency
> rather than only when there actually is one.
>
> 2. Inserting into a table or updating it only takes RowExclusiveLock,
> which is not even self-exclusive. I doubt that it's possible to change
> system catalogs in a concurrency-safe way with such a weak lock. For
> instance, if two sessions tried to do the same thing in concurrent
> transactions, they could both try to add the same dependency at the
> same time.
>
> 3. I'm not sure that CTAS, INSERT INTO...SELECT, and CREATE
> TABLE...EXECUTE are the only ways that datums can creep from one table
> into another. For example, what if I create a plpgsql function that
> gets a value from one table and stores it in a variable, and then use
> that variable to drive an INSERT into another table? I seem to recall
> there are complex cases involving records and range types and arrays,
> too, where the compressed object gets wrapped inside of another
> object; though maybe that wouldn't matter to your implementation if
> INSERT INTO ... SELECT uses a sufficiently aggressive strategy for
> adding dependencies.
+1 to all of these.
> I think we could add plain-old zstd compression without really
> tackling this issue
+1
> I'm now also curious to know whether Andres would agree that it's bad
> if zstd dictionaries are un-droppable. After all, I thought it would
> be bad if there was no way to eliminate a dependency on a compression
> method, and he disagreed.
I still am not too worried about that aspect. However:
> So maybe he would also think undroppable dictionaries are fine.
I'm much less sanguine about this. Imagine a schema based multi-tenancy setup,
where tenants come and go, and where a few of the tables use custom
dictionaries. Whereas not being able to get rid of lz4 at all has basically no
cost whatsoever, collecting more and more unusable dictionaries can imply a
fair amount of space usage after a while. I don't see any argument why that
would be ok, really.
> But maybe not. It seems even worse to me than undroppable compression
> methods, because you'll probably not have that many compression methods
> ever, but you could have a large number of dictionaries eventually.
Agreed on the latter.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-23 15:59 Robert Haas <[email protected]>
parent: Nikhil Kumar Veldanda <[email protected]>
0 siblings, 2 replies; 21+ messages in thread
From: Robert Haas @ 2025-04-23 15:59 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: pgsql-hackers
On Mon, Apr 21, 2025 at 8:52 PM Nikhil Kumar Veldanda
<[email protected]> wrote:
> After reviewing the email thread you attached on previous response, I
> identified a natural choke point for both inserts and updates: the
> call to "heap_toast_insert_or_update" inside
> heap_prepare_insert/heap_update. In the current master branch, that
> function only runs when HeapTupleHasExternal is true; my patch extends
> it to HeapTupleHasVarWidth tuples as well.
Isn't that basically all tuples, though? I think that's where this gets painful.
> On the performance side, my basic benchmarks show almost no regression
> for simple INSERT … VALUES workloads. CTAS, however, does regress
> noticeably: a CTAS completes in about 4 seconds before this patch, but
> with this patch it takes roughly 24 seconds. (For reference, a normal
> insert into the source table took about 58 seconds when using zstd
> dictionary compression), I suspect the extra cost comes from the added
> zstd decompression and PGLZ compression on the destination table.
That's nice to know, but I think the key question is not so much what
the feature costs when it is used but what it costs when it isn't
used. If we implement a system where we don't let
dictionary-compressed zstd datums leak out of tables, that's bound to
slow down a CTAS from a table where this feature is used, but that's
kind of OK: the feature has pros and cons, and if you don't like those
tradeoffs, you don't have to use it. However, it sounds like this
could also slow down inserts and updates in some cases even for users
who are not making use of the feature, and that's going to be a major
problem unless it can be shown that there is no case where the impact
is at all significant. Users hate paying for features that they aren't
using.
I wonder if there's a possible design where we only allow
dictionary-compressed datums to exist as top-level attributes in
designated tables to which those dictionaries are attached; and any
time you try to bury that Datum inside a container object (row, range,
array, whatever) detoasting is forced. If there's a clean and
inexpensive way to implement that, then you could avoid having
heap_toast_insert_or_update care about HeapTupleHasExternal(), which
seems like it might be a key point.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-23 16:27 Robert Haas <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 0 replies; 21+ messages in thread
From: Robert Haas @ 2025-04-23 16:27 UTC (permalink / raw)
To: Nikhil Kumar Veldanda <[email protected]>; +Cc: pgsql-hackers
On Wed, Apr 23, 2025 at 11:59 AM Robert Haas <[email protected]> wrote:
> heap_toast_insert_or_update care about HeapTupleHasExternal(), which
> seems like it might be a key point.
Care about HeapTupleHasVarWidth, rather.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: ZStandard (with dictionaries) compression support for TOAST compression
@ 2025-04-25 00:48 Michael Paquier <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 0 replies; 21+ messages in thread
From: Michael Paquier @ 2025-04-25 00:48 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nikhil Kumar Veldanda <[email protected]>; pgsql-hackers
On Wed, Apr 23, 2025 at 11:59:26AM -0400, Robert Haas wrote:
> That's nice to know, but I think the key question is not so much what
> the feature costs when it is used but what it costs when it isn't
> used. If we implement a system where we don't let
> dictionary-compressed zstd datums leak out of tables, that's bound to
> slow down a CTAS from a table where this feature is used, but that's
> kind of OK: the feature has pros and cons, and if you don't like those
> tradeoffs, you don't have to use it. However, it sounds like this
> could also slow down inserts and updates in some cases even for users
> who are not making use of the feature, and that's going to be a major
> problem unless it can be shown that there is no case where the impact
> is at all significant. Users hate paying for features that they aren't
> using.
The cost of digesting a dictionnary when decompressing sets of values
is also something I think we should worry about, FWIW (see [1]), as
the digesting cost is documented as costly, so I think that there is
also an argument in making the feature efficient if used. That would
hurt if a sequential scan needs to detoast multiple blobs with the
same dict. If we attach that on a per-value value, wouldn't it imply
that we need to digest the dictionnary every time a blob is
decompressed? This information could be cached, but it seems a bit
weird to me to invent a new level of relation caching for would could
be attached as a relation attribute option in the relcache. If a
dictionnary gets trained with a new sample of values, we could rely on
the invalidation to pass the new information.
Based on what I'm reading and I know very little about the topic so I
may be wrong, but does it even make sense to allow multiple
dictionnaries to be used in a single attribute? Of course that may
depend on the JSON blob patterns a single attribute is dealing with,
but I'm not sure that this is worth the extra complexity this creates.
> I wonder if there's a possible design where we only allow
> dictionary-compressed datums to exist as top-level attributes in
> designated tables to which those dictionaries are attached; and any
> time you try to bury that Datum inside a container object (row, range,
> array, whatever) detoasting is forced. If there's a clean and
> inexpensive way to implement that, then you could avoid having
> heap_toast_insert_or_update care about HeapTupleHasExternal(), which
> seems like it might be a key point.
Interesting, not sure.
FWIW, I'd still try to focus on making varatt more extensible with
plain zstd support first, because diving in all these details. We are
going to need it anyway.
[1]: https://facebook.github.io/zstd/zstd_manual.html#Chapter10
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 21+ messages in thread
end of thread, other threads:[~2025-04-25 00:48 UTC | newest]
Thread overview: 21+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-03-13 07:59 [PATCH v52 3/7] Make archiver process an auxiliary process Kyotaro Horiguchi <[email protected]>
2025-03-06 05:32 ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-03-06 12:02 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Kirill Reshke <[email protected]>
2025-03-06 13:35 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Aleksander Alekseev <[email protected]>
2025-03-06 16:47 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-03-06 19:15 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Robert Haas <[email protected]>
2025-03-06 19:33 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Tom Lane <[email protected]>
2025-03-06 22:36 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-03-07 11:42 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Aleksander Alekseev <[email protected]>
2025-03-08 01:35 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-03-08 16:28 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-03-17 20:02 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Robert Haas <[email protected]>
2025-04-15 18:13 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-04-18 16:22 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Robert Haas <[email protected]>
2025-04-21 07:02 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Michael Paquier <[email protected]>
2025-04-22 00:52 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Nikhil Kumar Veldanda <[email protected]>
2025-04-23 15:59 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Robert Haas <[email protected]>
2025-04-23 16:27 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Robert Haas <[email protected]>
2025-04-25 00:48 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Michael Paquier <[email protected]>
2025-04-22 16:24 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Andres Freund <[email protected]>
2025-03-07 11:56 ` Re: ZStandard (with dictionaries) compression support for TOAST compression Aleksander Alekseev <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox