public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 2/5] Quick-Vacuum-Strategy
103+ messages / 16 participants
[nested] [flat]
* [PATCH 2/5] Quick-Vacuum-Strategy
@ 2018-08-15 08:34 Andrey V. Lepikhov <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andrey V. Lepikhov @ 2018-08-15 08:34 UTC (permalink / raw)
---
src/backend/access/heap/heapam.c | 31 ++++++
src/backend/access/heap/pruneheap.c | 40 ++++++-
src/backend/commands/vacuumlazy.c | 161 +++++++++++++++++++++++++++-
src/backend/utils/misc/guc.c | 10 +-
src/include/access/heapam.h | 1 +
5 files changed, 234 insertions(+), 9 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 72395a50b8..dd033668b6 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -9392,6 +9392,36 @@ heap_sync(Relation rel)
}
}
+/*
+ * Mask DEAD tuples at a HEAP page
+ *
+ * We introduce this function for wal_consistency_checking satisfaction at
+ * Hot Standby node.
+ *
+ * DEAD tuple on a master node has t_cid and t_infomask, which can not be
+ * obtained by WAL records applying on a Hot Standby node.
+ */
+static void
+mask_dead_tuples(Page page)
+{
+ OffsetNumber offnum;
+
+ for (offnum = FirstOffsetNumber; offnum <= PageGetMaxOffsetNumber(page); offnum = OffsetNumberNext(offnum))
+ {
+ ItemId lp = PageGetItemId(page, offnum);
+ char* htup_begin;
+
+ if (!ItemIdIsDead(lp))
+ continue;
+
+ if (!ItemIdHasStorage(lp))
+ continue;
+
+ htup_begin = (char *) PageGetItem(page, lp);
+ memset(htup_begin, MASK_MARKER, ItemIdGetLength(lp));
+ }
+}
+
/*
* Mask a heap page before performing consistency checks on it.
*/
@@ -9405,6 +9435,7 @@ heap_mask(char *pagedata, BlockNumber blkno)
mask_page_hint_bits(page);
mask_unused_space(page);
+ mask_dead_tuples(page);
for (off = 1; off <= PageGetMaxOffsetNumber(page); off++)
{
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index c2f5343dac..ebbafe3275 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -43,6 +43,9 @@ typedef struct
bool marked[MaxHeapTuplesPerPage + 1];
} PruneState;
+/* Parameter for target deletion strategy in lazy vacuum */
+double target_index_deletion_factor = 0.01;
+
/* Local functions */
static int heap_prune_chain(Relation relation, Buffer buffer,
OffsetNumber rootoffnum,
@@ -405,7 +408,7 @@ heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rootoffnum,
if (HeapTupleSatisfiesVacuum(&tup, OldestXmin, buffer)
== HEAPTUPLE_DEAD && !HeapTupleHeaderIsHotUpdated(htup))
{
- heap_prune_record_unused(prstate, rootoffnum);
+ heap_prune_record_dead(prstate, rootoffnum);
HeapTupleHeaderAdvanceLatestRemovedXid(htup,
&prstate->latestRemovedXid);
ndeleted++;
@@ -580,8 +583,10 @@ heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rootoffnum,
*/
for (i = 1; (i < nchain) && (chainitems[i - 1] != latestdead); i++)
{
- heap_prune_record_unused(prstate, chainitems[i]);
ndeleted++;
+ if (chainitems[i] == latestdead)
+ continue;
+ heap_prune_record_unused(prstate, chainitems[i]);
}
/*
@@ -598,9 +603,28 @@ heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rootoffnum,
* redirect the root to the correct chain member.
*/
if (i >= nchain)
+ {
+ if (rootoffnum != latestdead)
+ {
+ if (ItemIdIsNormal(rootlp))
+ heap_prune_record_unused(prstate, latestdead);
+ else
+ {
+ /*
+ * We allow overlapping of redirected and dead items
+ */
+ heap_prune_record_redirect(prstate, rootoffnum, latestdead);
+ heap_prune_record_dead(prstate, latestdead);
+ }
+ }
heap_prune_record_dead(prstate, rootoffnum);
+ }
else
+ {
+ if (rootoffnum != latestdead)
+ heap_prune_record_unused(prstate, latestdead);
heap_prune_record_redirect(prstate, rootoffnum, chainitems[i]);
+ }
}
else if (nchain < 2 && ItemIdIsRedirected(rootlp))
{
@@ -653,7 +677,12 @@ heap_prune_record_dead(PruneState *prstate, OffsetNumber offnum)
Assert(prstate->ndead < MaxHeapTuplesPerPage);
prstate->nowdead[prstate->ndead] = offnum;
prstate->ndead++;
- Assert(!prstate->marked[offnum]);
+
+ /*
+ * We suppress checking prstate->marked[offnum]. It is not the best idea,
+ * but this is most simplistic way to enable Dead Redirecting by
+ * overlapping Dead and Redirected states.
+ */
prstate->marked[offnum] = true;
}
@@ -706,7 +735,10 @@ heap_page_prune_execute(Buffer buffer,
OffsetNumber off = *offnum++;
ItemId lp = PageGetItemId(page, off);
- ItemIdSetDead(lp);
+ if (target_index_deletion_factor > 0)
+ ItemIdMarkDead(lp);
+ else
+ ItemIdSetDead(lp);
}
/* Update all now-unused line pointers */
diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c
index 5649a70800..9a625b8dba 100644
--- a/src/backend/commands/vacuumlazy.c
+++ b/src/backend/commands/vacuumlazy.c
@@ -41,13 +41,16 @@
#include "access/heapam_xlog.h"
#include "access/htup_details.h"
#include "access/multixact.h"
+#include "access/nbtree.h"
#include "access/transam.h"
#include "access/visibilitymap.h"
#include "access/xlog.h"
+#include "catalog/index.h"
#include "catalog/storage.h"
#include "commands/dbcommands.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
+#include "executor/executor.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "portability/instr_time.h"
@@ -139,7 +142,6 @@ typedef struct LVRelStats
bool lock_waiter_detected;
} LVRelStats;
-
/* A few variables that don't seem worth passing around as parameters */
static int elevel = -1;
@@ -156,6 +158,8 @@ static void lazy_scan_heap(Relation onerel, int options,
bool aggressive);
static void lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats);
static bool lazy_check_needs_freeze(Buffer buf, bool *hastup);
+static void quick_vacuum_index(Relation irel, Relation hrel,
+ LVRelStats *vacrelstats);
static void lazy_vacuum_index(Relation indrel,
IndexBulkDeleteResult **stats,
LVRelStats *vacrelstats);
@@ -734,10 +738,17 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
/* Remove index entries */
for (i = 0; i < nindexes; i++)
- lazy_vacuum_index(Irel[i],
+ {
+ bool use_quick_strategy = (vacrelstats->num_dead_tuples/vacrelstats->old_live_tuples < target_index_deletion_factor);
+
+ if (use_quick_strategy && (Irel[i]->rd_amroutine->amtargetdelete != NULL))
+ quick_vacuum_index(Irel[i], onerel, vacrelstats);
+ else
+ lazy_vacuum_index(Irel[i],
&indstats[i],
vacrelstats);
+ }
/*
* Report that we are now vacuuming the heap. We also increase
* the number of index scans here; note that by using
@@ -1379,10 +1390,16 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
/* Remove index entries */
for (i = 0; i < nindexes; i++)
- lazy_vacuum_index(Irel[i],
+ {
+ bool use_quick_strategy = (vacrelstats->num_dead_tuples/vacrelstats->old_live_tuples < target_index_deletion_factor);
+
+ if (use_quick_strategy && (Irel[i]->rd_amroutine->amtargetdelete != NULL))
+ quick_vacuum_index(Irel[i], onerel, vacrelstats);
+ else
+ lazy_vacuum_index(Irel[i],
&indstats[i],
vacrelstats);
-
+ }
/* Report that we are now vacuuming the heap */
hvp_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_HEAP;
hvp_val[1] = vacrelstats->num_index_scans + 1;
@@ -1671,6 +1688,142 @@ lazy_check_needs_freeze(Buffer buf, bool *hastup)
return false;
}
+/*
+ * Get tuple from heap for a scan key building.
+ */
+static HeapTuple
+get_tuple_by_tid(Relation rel, ItemPointer tid)
+{
+ Buffer buffer;
+ Page page;
+ OffsetNumber offnum;
+ ItemId lp;
+ HeapTuple tuple;
+ bool needLock = !RELATION_IS_LOCAL(rel);
+ BlockNumber npages;
+
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ npages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+ if (ItemPointerGetBlockNumber(tid) > npages)
+ return NULL;
+
+ buffer = ReadBufferExtended(rel, MAIN_FORKNUM, ItemPointerGetBlockNumber(tid), RBM_NORMAL, NULL);
+ LockBuffer(buffer, BUFFER_LOCK_SHARE);
+
+ page = (Page) BufferGetPage(buffer);
+ offnum = ItemPointerGetOffsetNumber(tid);
+ lp = PageGetItemId(page, offnum);
+
+ /*
+ * VACUUM Races: someone already remove the tuple from HEAP. Ignore it.
+ */
+ if (!ItemIdIsUsed(lp))
+ {
+ UnlockReleaseBuffer(buffer);
+ return NULL;
+ }
+ /* Walk along the chain */
+ while (!ItemIdHasStorage(lp))
+ {
+ offnum = ItemIdGetRedirect(lp);
+ lp = PageGetItemId(page, offnum);
+ Assert(ItemIdIsUsed(lp));
+ }
+
+ /* Form a tuple */
+ tuple = palloc(sizeof(HeapTupleData));
+ ItemPointerSet(&(tuple->t_self), BufferGetBlockNumber(buffer), offnum);
+ tuple->t_tableOid = RelationGetRelid(rel);
+ tuple->t_data = (HeapTupleHeader) PageGetItem(page, lp);
+ tuple->t_len = ItemIdGetLength(lp);
+ UnlockReleaseBuffer(buffer);
+ return tuple;
+}
+
+/*
+ * quick_vacuum_index() -- quick vacuum one index relation.
+ *
+ * Delete all the index entries pointing to tuples listed in
+ * vacrelstats->dead_tuples.
+ */
+static void
+quick_vacuum_index(Relation irel, Relation hrel,
+ LVRelStats *vacrelstats)
+{
+ int tnum;
+ bool* found = palloc0(vacrelstats->num_dead_tuples*sizeof(bool));
+ IndexInfo* indexInfo = BuildIndexInfo(irel);
+ EState* estate = CreateExecutorState();
+ ExprContext* econtext = GetPerTupleExprContext(estate);
+ ExprState* predicate = ExecPrepareQual(indexInfo->ii_Predicate, estate);
+ TupleTableSlot* slot = MakeSingleTupleTableSlot(RelationGetDescr(hrel));
+
+ IndexTargetDeleteResult stats;
+ IndexTargetDeleteInfo ivinfo;
+
+ ivinfo.indexRelation = irel;
+ ivinfo.heapRelation = hrel;
+
+ econtext->ecxt_scantuple = slot;
+
+ /* Get tuple from heap */
+ for (tnum = 0; tnum < vacrelstats->num_dead_tuples; tnum++)
+ {
+ HeapTuple tuple;
+ Datum values[INDEX_MAX_KEYS];
+ bool isnull[INDEX_MAX_KEYS];
+
+ /* Index entry for the TID was deleted early */
+ if (found[tnum])
+ continue;
+
+ /* Get a tuple from heap */
+ if ((tuple = get_tuple_by_tid(hrel, &(vacrelstats->dead_tuples[tnum]))) == NULL)
+ {
+ /*
+ * Tuple has 'not used' status.
+ */
+ found[tnum] = true;
+ continue;
+ }
+
+ /*
+ * Form values[] and isnull[] arrays from for index tuple
+ * by heap tuple
+ */
+ MemoryContextReset(econtext->ecxt_per_tuple_memory);
+
+ ExecStoreTuple(tuple, slot, InvalidBuffer, false);
+
+ /*
+ * In a partial index, ignore tuples that don't satisfy the
+ * predicate.
+ */
+ if ((predicate != NULL) && (!ExecQual(predicate, econtext)))
+ {
+ found[tnum] = true;
+ continue;
+ }
+
+ FormIndexDatum(indexInfo, slot, estate, values, isnull);
+
+ /*
+ * Make attempt to delete some index entries by one tree descent.
+ * We use only a part of TID list, which contains not found TID's.
+ */
+ ivinfo.dead_tuples = &(vacrelstats->dead_tuples[tnum]);
+ ivinfo.num_dead_tuples = vacrelstats->num_dead_tuples - tnum;
+ ivinfo.found_dead_tuples = found + tnum;
+ index_target_delete(&ivinfo, &stats, values, isnull);
+ }
+
+ pfree(found);
+ ExecDropSingleTupleTableSlot(slot);
+ FreeExecutorState(estate);
+}
/*
* lazy_vacuum_index() -- vacuum one index relation.
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index a88ea6cfc9..e9e953ece6 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3243,7 +3243,15 @@ static struct config_real ConfigureNamesReal[] =
0.1, 0.0, 100.0,
NULL, NULL, NULL
},
-
+ {
+ {"target_index_deletion_factor", PGC_SIGHUP, AUTOVACUUM,
+ gettext_noop("Maximum number of vacuumed tuples as a fraction of reltuples where we can use target index vacuum strategy."),
+ NULL
+ },
+ &target_index_deletion_factor,
+ 0.01, 0.0, 1.0,
+ NULL, NULL, NULL
+ },
{
{"checkpoint_completion_target", PGC_SIGHUP, WAL_CHECKPOINTS,
gettext_noop("Time spent flushing dirty buffers during checkpoint, as fraction of checkpoint interval."),
diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h
index ca5cad7497..c5a7bc8361 100644
--- a/src/include/access/heapam.h
+++ b/src/include/access/heapam.h
@@ -100,6 +100,7 @@ extern Relation heap_openrv_extended(const RangeVar *relation,
typedef struct HeapScanDescData *HeapScanDesc;
typedef struct ParallelHeapScanDescData *ParallelHeapScanDesc;
+extern double target_index_deletion_factor;
/*
* HeapScanIsValid
* True iff the heap scan is valid.
--
2.17.1
--------------37834DEDB242246EE7291CE9
Content-Type: text/x-patch;
name="0003-Make-all-nbtree-index-tuples-have-unique-keys.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0003-Make-all-nbtree-index-tuples-have-unique-keys.patch"
^ permalink raw reply [nested|flat] 103+ messages in thread
* In-placre persistance change of a relation
@ 2020-11-11 08:33 Kyotaro Horiguchi <[email protected]>
2020-11-11 22:18 ` Re: In-placre persistance change of a relation Andres Freund <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 2 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-11 08:33 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hello. This is a thread for an alternative solution to wal_level=none
[*1] for bulk data loading.
*1: https://www.postgresql.org/message-id/TYAPR01MB29901EBE5A3ACCE55BA99186FE320%40TYAPR01MB2990.jpnprd0...
At Tue, 10 Nov 2020 09:33:12 -0500, Stephen Frost <[email protected]> wrote in
> Greetings,
>
> * Kyotaro Horiguchi ([email protected]) wrote:
> > For fuel(?) of the discussion, I tried a very-quick PoC for in-place
> > ALTER TABLE SET LOGGED/UNLOGGED and resulted as attached. After some
> > trials of several ways, I drifted to the following way after poking
> > several ways.
> >
> > 1. Flip BM_PERMANENT of active buffers
> > 2. adding/removing init fork
> > 3. sync files,
> > 4. Flip pg_class.relpersistence.
> >
> > It always skips table copy in the SET UNLOGGED case, and only when
> > wal_level=minimal in the SET LOGGED case. Crash recovery seems
> > working by some brief testing by hand.
>
> Somehow missed that this patch more-or-less does what I was referring to
> down-thread, but I did want to mention that it looks like it's missing a
> necessary FlushRelationBuffers() call before the sync, otherwise there
> could be dirty buffers for the relation that's being set to LOGGED (with
> wal_level=minimal), which wouldn't be good. See the comments above
> smgrimmedsync().
Right. Thanks. However, since SetRelFileNodeBuffersPersistence()
called just above scans shared buffers so I don't want to just call
FlushRelationBuffers() separately. Instead, I added buffer-flush to
SetRelFileNodeBuffersPersistence().
FWIW this is a revised version of the PoC, which has some known
problems.
- Flipping of Buffer persistence is not WAL-logged nor even be able to
be safely roll-backed. (It might be better to drop buffers).
- This version handles indexes but not yet handle toast relatins.
- tableAMs are supposed to support this feature. (but I'm not sure
it's worth allowing them not to do so).
> > Of course, I haven't performed intensive test on it.
>
> Reading through the thread, it didn't seem very clear, but we should
> definitely make sure that it does the right thing on replicas when going
> between unlogged and logged (and between logged and unlogged too), of
> course.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
Attachments:
[text/x-patch] PoC_in-place_set_persistence_v2.patch (23.8K, ../../[email protected]/2-PoC_in-place_set_persistence_v2.patch)
download | inline diff:
diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c
index dcaea7135f..0c6ce70484 100644
--- a/src/backend/access/heap/heapam_handler.c
+++ b/src/backend/access/heap/heapam_handler.c
@@ -613,6 +613,27 @@ heapam_relation_set_new_filenode(Relation rel,
smgrclose(srel);
}
+static void
+heapam_relation_set_persistence(Relation rel, char persistence)
+{
+ Assert(rel->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT ||
+ rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED);
+
+ Assert (rel->rd_rel->relpersistence != persistence);
+
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ {
+ Assert(rel->rd_rel->relkind == RELKIND_RELATION ||
+ rel->rd_rel->relkind == RELKIND_MATVIEW ||
+ rel->rd_rel->relkind == RELKIND_TOASTVALUE);
+
+ RelationCreateInitFork(rel->rd_node, false);
+ }
+ else
+ RelationDropInitFork(rel->rd_node);
+}
+
+
static void
heapam_relation_nontransactional_truncate(Relation rel)
{
@@ -2540,6 +2561,7 @@ static const TableAmRoutine heapam_methods = {
.compute_xid_horizon_for_tuples = heap_compute_xid_horizon_for_tuples,
.relation_set_new_filenode = heapam_relation_set_new_filenode,
+ .relation_set_persistence = heapam_relation_set_persistence,
.relation_nontransactional_truncate = heapam_relation_nontransactional_truncate,
.relation_copy_data = heapam_relation_copy_data,
.relation_copy_for_cluster = heapam_relation_copy_for_cluster,
diff --git a/src/backend/access/rmgrdesc/smgrdesc.c b/src/backend/access/rmgrdesc/smgrdesc.c
index a7c0cb1bc3..8397002613 100644
--- a/src/backend/access/rmgrdesc/smgrdesc.c
+++ b/src/backend/access/rmgrdesc/smgrdesc.c
@@ -40,6 +40,14 @@ smgr_desc(StringInfo buf, XLogReaderState *record)
xlrec->blkno, xlrec->flags);
pfree(path);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) rec;
+ char *path = relpathperm(xlrec->rnode, xlrec->forkNum);
+
+ appendStringInfoString(buf, path);
+ pfree(path);
+ }
}
const char *
@@ -55,6 +63,9 @@ smgr_identify(uint8 info)
case XLOG_SMGR_TRUNCATE:
id = "TRUNCATE";
break;
+ case XLOG_SMGR_UNLINK:
+ id = "UNLINK";
+ break;
}
return id;
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index d538f25726..ac5aea3d38 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -60,6 +60,8 @@ int wal_skip_threshold = 2048; /* in kilobytes */
typedef struct PendingRelDelete
{
RelFileNode relnode; /* relation that may need to be deleted */
+ bool deleteinitfork; /* delete only init fork if true */
+ bool createinitfork; /* create init fork if true */
BackendId backend; /* InvalidBackendId if not a temp rel */
bool atCommit; /* T=delete at commit; F=delete at abort */
int nestLevel; /* xact nesting level of request */
@@ -153,6 +155,8 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rnode;
+ pending->deleteinitfork = false;
+ pending->createinitfork = false;
pending->backend = backend;
pending->atCommit = false; /* delete if abort */
pending->nestLevel = GetCurrentTransactionNestLevel();
@@ -168,6 +172,95 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
return srel;
}
+/*
+ * RelationCreateInitFork
+ * Create physical storage for a relation.
+ *
+ * Create the underlying disk file storage for the relation. This only
+ * creates the main fork; additional forks are created lazily by the
+ * modules that need them.
+ *
+ * This function is transactional. The creation is WAL-logged, and if the
+ * transaction aborts later on, the storage will be destroyed.
+ */
+void
+RelationCreateInitFork(RelFileNode rnode, bool isRedo)
+{
+ PendingRelDelete *pending;
+ SMgrRelation srel;
+ PendingRelDelete *prev;
+ PendingRelDelete *next;
+
+ prev = NULL;
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ pending->deleteinitfork && pending->atCommit)
+ {
+ /* unlink and delete list entry */
+ if (prev)
+ prev->next = next;
+ else
+ pendingDeletes = next;
+ pfree(pending);
+ return;
+ }
+ else
+ {
+ /* unrelated entry, don't touch it */
+ prev = pending;
+ }
+ }
+ srel = smgropen(rnode, InvalidBackendId);
+ smgrcreate(srel, INIT_FORKNUM, isRedo);
+ if (!isRedo)
+ log_smgrcreate(&rnode, INIT_FORKNUM);
+ smgrimmedsync(srel, INIT_FORKNUM);
+
+ /* Add the relation to the list of stuff to delete at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->deleteinitfork = true;
+ pending->createinitfork = false;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false; /* delete if abort */
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
+void
+RelationDropInitFork(RelFileNode rnode)
+{
+ PendingRelDelete *pending;
+ PendingRelDelete *next;
+
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ pending->deleteinitfork && pending->atCommit)
+ {
+ /* We're done. */
+ return;
+ }
+ }
+
+ /* Add the relation to the list of stuff to delete at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->deleteinitfork = true;
+ pending->createinitfork = false;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true; /* create if abort */
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
/*
* Perform XLogInsert of an XLOG_SMGR_CREATE record to WAL.
*/
@@ -187,6 +280,25 @@ log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum)
XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE);
}
+/*
+ * Perform XLogInsert of an XLOG_SMGR_UNLINK record to WAL.
+ */
+void
+log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum)
+{
+ xl_smgr_unlink xlrec;
+
+ /*
+ * Make an XLOG entry reporting the file unlink.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.forkNum = forkNum;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_UNLINK | XLR_SPECIAL_REL_UPDATE);
+}
+
/*
* RelationDropStorage
* Schedule unlinking of physical storage at transaction commit.
@@ -200,6 +312,8 @@ RelationDropStorage(Relation rel)
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rel->rd_node;
+ pending->createinitfork = false;
+ pending->deleteinitfork = false;
pending->backend = rel->rd_backend;
pending->atCommit = true; /* delete if commit */
pending->nestLevel = GetCurrentTransactionNestLevel();
@@ -626,19 +740,27 @@ smgrDoPendingDeletes(bool isCommit)
srel = smgropen(pending->relnode, pending->backend);
- /* allocate the initial array, or extend it, if needed */
- if (maxrels == 0)
+ if (pending->deleteinitfork)
{
- maxrels = 8;
- srels = palloc(sizeof(SMgrRelation) * maxrels);
+ log_smgrunlink(&pending->relnode, INIT_FORKNUM);
+ smgrunlink(srel, INIT_FORKNUM, false);
}
- else if (maxrels <= nrels)
+ else
{
- maxrels *= 2;
- srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
- }
+ /* allocate the initial array, or extend it, if needed */
+ if (maxrels == 0)
+ {
+ maxrels = 8;
+ srels = palloc(sizeof(SMgrRelation) * maxrels);
+ }
+ else if (maxrels <= nrels)
+ {
+ maxrels *= 2;
+ srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
+ }
- srels[nrels++] = srel;
+ srels[nrels++] = srel;
+ }
}
/* must explicitly free the list entry */
pfree(pending);
@@ -917,6 +1039,14 @@ smgr_redo(XLogReaderState *record)
reln = smgropen(xlrec->rnode, InvalidBackendId);
smgrcreate(reln, xlrec->forkNum, true);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) XLogRecGetData(record);
+ SMgrRelation reln;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ smgrunlink(reln, xlrec->forkNum, true);
+ }
else if (info == XLOG_SMGR_TRUNCATE)
{
xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index e3cfaf8b07..e358174b01 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -4918,6 +4918,137 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel,
return newcmd;
}
+static bool
+try_inplace_persistence_change(AlteredTableInfo *tab, char persistence,
+ LOCKMODE lockmode)
+{
+ Relation rel;
+ Relation classRel;
+ HeapTuple tuple,
+ newtuple;
+ Datum new_val[Natts_pg_class];
+ bool new_null[Natts_pg_class],
+ new_repl[Natts_pg_class];
+ int i;
+ List *relids;
+ ListCell *lc_oid;
+
+ Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE);
+ Assert(lockmode == AccessExclusiveLock);
+
+ /*
+ * Under the following condition, we need to call ATRewriteTable, which
+ * cannot be false in the AT_REWRITE_ALTER_PERSISTENCE case.
+ */
+ Assert(tab->constraints == NULL && tab->partition_constraint == NULL &&
+ tab->newvals == NULL && !tab->verify_new_notnull);
+
+ /*
+ * When wal_level is replica or higher we need that the initial state of
+ * the relation be recoverable from WAL. When wal_level >= replica
+ * switching to PERMANENT needs to emit the WAL records to reconstruct the
+ * current data. This could be done by writing XLOG_FPI for all pages but
+ * it is not obvious that that is performant than normal rewriting.
+ * Otherwise what we need for the relation data is just establishing
+ * initial state on storage and no need of WAL to reconstruct it.
+ */
+ if (tab->newrelpersistence == RELPERSISTENCE_PERMANENT && XLogIsNeeded())
+ return false;
+
+ rel = table_open(tab->relid, lockmode);
+
+ Assert(rel->rd_rel->relpersistence != persistence);
+
+ elog(DEBUG1, "perform im-place persistnce change");
+
+ RelationOpenSmgr(rel);
+
+ /* Change persistence then flush-out buffers of the relation */
+
+ /* Get the list of index OIDs for this relation */
+ relids = RelationGetIndexList(rel);
+ relids = lcons_oid(rel->rd_id, relids);
+
+ table_close(rel, lockmode);
+
+ /* Done change on storage. Update catalog including indexes. */
+ /* add the heap oid to the relation ID list */
+
+ classRel = table_open(RelationRelationId, RowExclusiveLock);
+
+ foreach (lc_oid, relids)
+ {
+ Oid reloid = lfirst_oid(lc_oid);
+ Relation r = relation_open(reloid, lockmode);
+
+ RelationOpenSmgr(r);
+
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ {
+ RelationCreateInitFork(r->rd_node, false);
+
+ if (r->rd_rel->relkind == RELKIND_INDEX ||
+ r->rd_rel->relkind == RELKIND_PARTITIONED_INDEX)
+ r->rd_indam->ambuildempty(r);
+ else
+ {
+ Assert(r->rd_rel->relkind == RELKIND_RELATION ||
+ r->rd_rel->relkind == RELKIND_MATVIEW ||
+ r->rd_rel->relkind == RELKIND_TOASTVALUE);
+ }
+ }
+ else
+ RelationDropInitFork(r->rd_node);
+
+ table_close(r, NoLock);
+
+ /*
+ * This relation is now WAL-logged. Sync all files immediately to
+ * establish the initial state on storgae.
+ */
+ if (persistence == RELPERSISTENCE_PERMANENT)
+ {
+ for (i = 0 ; i < MAX_FORKNUM ; i++)
+ {
+ if (smgrexists(r->rd_smgr, i))
+ smgrimmedsync(r->rd_smgr, i);
+ }
+ }
+
+
+ tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+ if (!HeapTupleIsValid(tuple))
+ elog(ERROR, "cache lookup failed for relation %u", reloid);
+
+ memset(new_val, 0, sizeof(new_val));
+ memset(new_null, false, sizeof(new_null));
+ memset(new_repl, false, sizeof(new_repl));
+
+ new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence);
+ new_null[Anum_pg_class_relpersistence - 1] = false;
+ new_repl[Anum_pg_class_relpersistence - 1] = true;
+
+ newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
+ new_val, new_null, new_repl);
+
+ CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+ }
+
+ foreach (lc_oid, relids)
+ {
+ Oid reloid = lfirst_oid(lc_oid);
+ Relation r = relation_open(reloid, lockmode);
+
+ RelationOpenSmgr(r);
+ SetRelationBuffersPersistence(r, persistence == RELPERSISTENCE_PERMANENT);
+ table_close(r, NoLock);
+ }
+ table_close(classRel, RowExclusiveLock);
+
+ return true;
+}
+
/*
* ATRewriteTables: ALTER TABLE phase 3
*/
@@ -5038,45 +5169,51 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode,
tab->relid,
tab->rewrite);
- /*
- * Create transient table that will receive the modified data.
- *
- * Ensure it is marked correctly as logged or unlogged. We have
- * to do this here so that buffers for the new relfilenode will
- * have the right persistence set, and at the same time ensure
- * that the original filenode's buffers will get read in with the
- * correct setting (i.e. the original one). Otherwise a rollback
- * after the rewrite would possibly result with buffers for the
- * original filenode having the wrong persistence setting.
- *
- * NB: This relies on swap_relation_files() also swapping the
- * persistence. That wouldn't work for pg_class, but that can't be
- * unlogged anyway.
- */
- OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, persistence,
- lockmode);
+ if (tab->rewrite != AT_REWRITE_ALTER_PERSISTENCE ||
+ !try_inplace_persistence_change(tab, persistence, lockmode))
+ {
+ /*
+ * Create transient table that will receive the modified data.
+ *
+ * Ensure it is marked correctly as logged or unlogged. We
+ * have to do this here so that buffers for the new relfilenode
+ * will have the right persistence set, and at the same time
+ * ensure that the original filenode's buffers will get read in
+ * with the correct setting (i.e. the original one). Otherwise
+ * a rollback after the rewrite would possibly result with
+ * buffers for the original filenode having the wrong
+ * persistence setting.
+ *
+ * NB: This relies on swap_relation_files() also swapping the
+ * persistence. That wouldn't work for pg_class, but that can't
+ * be unlogged anyway.
+ */
+ OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, persistence,
+ lockmode);
- /*
- * Copy the heap data into the new table with the desired
- * modifications, and test the current data within the table
- * against new constraints generated by ALTER TABLE commands.
- */
- ATRewriteTable(tab, OIDNewHeap, lockmode);
+ /*
+ * Copy the heap data into the new table with the desired
+ * modifications, and test the current data within the table
+ * against new constraints generated by ALTER TABLE commands.
+ */
+ ATRewriteTable(tab, OIDNewHeap, lockmode);
- /*
- * Swap the physical files of the old and new heaps, then rebuild
- * indexes and discard the old heap. We can use RecentXmin for
- * the table's new relfrozenxid because we rewrote all the tuples
- * in ATRewriteTable, so no older Xid remains in the table. Also,
- * we never try to swap toast tables by content, since we have no
- * interest in letting this code work on system catalogs.
- */
- finish_heap_swap(tab->relid, OIDNewHeap,
- false, false, true,
- !OidIsValid(tab->newTableSpace),
- RecentXmin,
- ReadNextMultiXactId(),
- persistence);
+ /*
+ * Swap the physical files of the old and new heaps, then
+ * rebuild indexes and discard the old heap. We can use
+ * RecentXmin for the table's new relfrozenxid because we
+ * rewrote all the tuples in ATRewriteTable, so no older Xid
+ * remains in the table. Also, we never try to swap toast
+ * tables by content, since we have no interest in letting this
+ * code work on system catalogs.
+ */
+ finish_heap_swap(tab->relid, OIDNewHeap,
+ false, false, true,
+ !OidIsValid(tab->newTableSpace),
+ RecentXmin,
+ ReadNextMultiXactId(),
+ persistence);
+ }
}
else
{
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index ad0d1a9abc..c71e1a5f92 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -3033,6 +3033,80 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
}
}
+/* ---------------------------------------------------------------------
+ * SetRelFileNodeBuffersPersistence
+ *
+ * This function changes the persistence of all buffer pages of a relation
+ * then writes all dirty pages of the relation out to disk when switching
+ * to PERMANENT. (or more accurately, out to kernel disk buffers),
+ * ensuring that the kernel has an up-to-date view of the relation.
+ *
+ * Generally, the caller should be holding AccessExclusiveLock on the
+ * target relation to ensure that no other backend is busy dirtying
+ * more blocks of the relation; the effects can't be expected to last
+ * after the lock is released.
+ *
+ * XXX currently it sequentially searches the buffer pool, should be
+ * changed to more clever ways of searching. This routine is not
+ * used in any performance-critical code paths, so it's not worth
+ * adding additional overhead to normal paths to make it go faster;
+ * but see also DropRelFileNodeBuffers.
+ * --------------------------------------------------------------------
+ */
+void
+SetRelationBuffersPersistence(Relation rel, bool permanent)
+{
+ int i;
+ RelFileNodeBackend rnode = rel->rd_smgr->smgr_rnode;
+
+ Assert (!RelFileNodeBackendIsTemp(rnode));
+
+ for (i = 0; i < NBuffers; i++)
+ {
+ BufferDesc *bufHdr = GetBufferDescriptor(i);
+ uint32 buf_state;
+
+ if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ continue;
+
+ ReservePrivateRefCountEntry();
+
+ buf_state = LockBufHdr(bufHdr);
+
+ if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ {
+ ereport(LOG, (errmsg ("#%d: %d", i, (buf_state & BM_PERMANENT) == 0), errhidestmt(true)));
+ if (permanent)
+ {
+ Assert ((buf_state & BM_PERMANENT) == 0);
+ buf_state |= BM_PERMANENT;
+ pg_atomic_write_u32(&bufHdr->state, buf_state);
+
+ /* we flush this buffer when swithing to PERMANENT */
+ if ((buf_state & (BM_VALID | BM_DIRTY)) ==
+ (BM_VALID | BM_DIRTY))
+ {
+ PinBuffer_Locked(bufHdr);
+ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr),
+ LW_SHARED);
+ FlushBuffer(bufHdr, rel->rd_smgr);
+ LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
+ UnpinBuffer(bufHdr, true);
+ }
+ else
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ else
+ {
+ Assert ((buf_state & BM_PERMANENT) != 0);
+ buf_state &= ~BM_PERMANENT;
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ ereport(LOG, (errmsg ("#%d: -> %d", i, (buf_state & BM_PERMANENT) == 0), errhidestmt(true)));
+ }
+ }
+}
+
/* ---------------------------------------------------------------------
* DropRelFileNodesAllBuffers
*
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index dcc09df0c7..5eb9e97b3d 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -645,6 +645,12 @@ smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
smgrsw[reln->smgr_which].smgr_immedsync(reln, forknum);
}
+void
+smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_unlink(reln->smgr_rnode, forknum, isRedo);
+}
+
/*
* AtEOXact_SMgr
*
diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h
index 387eb34a61..1d19278a18 100644
--- a/src/include/access/tableam.h
+++ b/src/include/access/tableam.h
@@ -451,6 +451,15 @@ typedef struct TableAmRoutine
TransactionId *freezeXid,
MultiXactId *minmulti);
+ /*
+ * This callback needs to switch persistence of the relation between
+ * RELPERSISTENCE_PERMANENT and RELPERSISTENCE_UNLOGGED. Actual change on
+ * storage is performed elsewhere.
+ *
+ * See also table_relation_set_persistence().
+ */
+ void (*relation_set_persistence) (Relation rel, char persistence);
+
/*
* This callback needs to remove all contents from `rel`'s current
* relfilenode. No provisions for transactional behaviour need to be made.
@@ -1404,6 +1413,18 @@ table_relation_set_new_filenode(Relation rel,
freezeXid, minmulti);
}
+/*
+ * Switch storage persistence between RELPERSISTENCE_PERMANENT and
+ * RELPERSISTENCE_UNLOGGED.
+ *
+ * This is used during in-place persistence switching
+ */
+static inline void
+table_relation_set_persistence(Relation rel, char persistence)
+{
+ rel->rd_tableam->relation_set_persistence(rel, persistence);
+}
+
/*
* Remove all table contents from `rel`, in a non-transactional manner.
* Non-transactional meaning that there's no need to support rollbacks. This
diff --git a/src/include/catalog/storage.h b/src/include/catalog/storage.h
index 30c38e0ca6..43d2eb0fb4 100644
--- a/src/include/catalog/storage.h
+++ b/src/include/catalog/storage.h
@@ -23,6 +23,8 @@
extern int wal_skip_threshold;
extern SMgrRelation RelationCreateStorage(RelFileNode rnode, char relpersistence);
+extern void RelationCreateInitFork(RelFileNode rel, bool isRedo);
+extern void RelationDropInitFork(RelFileNode rel);
extern void RelationDropStorage(Relation rel);
extern void RelationPreserveStorage(RelFileNode rnode, bool atCommit);
extern void RelationPreTruncate(Relation rel);
diff --git a/src/include/catalog/storage_xlog.h b/src/include/catalog/storage_xlog.h
index 7b21cab2e0..73ad2ae89e 100644
--- a/src/include/catalog/storage_xlog.h
+++ b/src/include/catalog/storage_xlog.h
@@ -29,6 +29,7 @@
/* XLOG gives us high 4 bits */
#define XLOG_SMGR_CREATE 0x10
#define XLOG_SMGR_TRUNCATE 0x20
+#define XLOG_SMGR_UNLINK 0x30
typedef struct xl_smgr_create
{
@@ -36,6 +37,12 @@ typedef struct xl_smgr_create
ForkNumber forkNum;
} xl_smgr_create;
+typedef struct xl_smgr_unlink
+{
+ RelFileNode rnode;
+ ForkNumber forkNum;
+} xl_smgr_unlink;
+
/* flags for xl_smgr_truncate */
#define SMGR_TRUNCATE_HEAP 0x0001
#define SMGR_TRUNCATE_VM 0x0002
@@ -51,6 +58,7 @@ typedef struct xl_smgr_truncate
} xl_smgr_truncate;
extern void log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum);
+extern void log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum);
extern void smgr_redo(XLogReaderState *record);
extern void smgr_desc(StringInfo buf, XLogReaderState *record);
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index ee91b8fa26..f65a273999 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -205,6 +205,7 @@ extern void FlushRelationsAllBuffers(struct SMgrRelationData **smgrs, int nrels)
extern void FlushDatabaseBuffers(Oid dbid);
extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
int nforks, BlockNumber *firstDelBlock);
+extern void SetRelationBuffersPersistence(Relation rnode, bool permanent);
extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
extern void DropDatabaseBuffers(Oid dbid);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index f28a842401..5d74631006 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -86,6 +86,7 @@ extern void smgrclose(SMgrRelation reln);
extern void smgrcloseall(void);
extern void smgrclosenode(RelFileNodeBackend rnode);
extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo);
extern void smgrdosyncall(SMgrRelation *rels, int nrels);
extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2020-11-11 22:18 ` Andres Freund <[email protected]>
2020-11-12 06:55 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Andres Freund @ 2020-11-11 22:18 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hi,
I suggest outlining what you are trying to achieve here. Starting a new
thread and expecting people to dig through another thread to infer what
you are actually trying to achive isn't great.
FWIW, I'm *extremely* doubtful it's worth adding features that depend on
a PGC_POSTMASTER wal_level=minimal being used. Which this does, a far as
I understand. If somebody added support for dynamically adapting
wal_level (e.g. wal_level=auto, that increases wal_level to
replica/logical depending on the presence of replication slots), it'd
perhaps be different.
On 2020-11-11 17:33:17 +0900, Kyotaro Horiguchi wrote:
> FWIW this is a revised version of the PoC, which has some known
> problems.
>
> - Flipping of Buffer persistence is not WAL-logged nor even be able to
> be safely roll-backed. (It might be better to drop buffers).
That's obviously a no-go. I think you might be able to address this if
you accept that the command cannot be run in a transaction (like
CONCURRENTLY). Then you can first do the catalog changes, change the
persistence level, and commit.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-11 22:18 ` Re: In-placre persistance change of a relation Andres Freund <[email protected]>
@ 2020-11-12 06:55 ` Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-12 06:55 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Wed, 11 Nov 2020 14:18:04 -0800, Andres Freund <[email protected]> wrote in
> Hi,
>
> I suggest outlining what you are trying to achieve here. Starting a new
> thread and expecting people to dig through another thread to infer what
> you are actually trying to achive isn't great.
Agreed. I'll post that. Thanks.
> FWIW, I'm *extremely* doubtful it's worth adding features that depend on
> a PGC_POSTMASTER wal_level=minimal being used. Which this does, a far as
> I understand. If somebody added support for dynamically adapting
> wal_level (e.g. wal_level=auto, that increases wal_level to
> replica/logical depending on the presence of replication slots), it'd
> perhaps be different.
Yes, this depends on wal_level=minimal for switching from UNLOGGED to
LOGGED, that's similar to COPY/INSERT-to-intransaction-created-tables
optimization for wal_level=minimal. And it expands that optimization
to COPY/INSERT-to-existent-tables, which seems worth doing.
Switching to LOGGED needs to emit the initial state to WAL... Hmm.. I
came to think that even in that case skipping table copy reduces I/O
significantly, even though FPI-WAL is emitted.
> On 2020-11-11 17:33:17 +0900, Kyotaro Horiguchi wrote:
> > FWIW this is a revised version of the PoC, which has some known
> > problems.
> >
> > - Flipping of Buffer persistence is not WAL-logged nor even be able to
> > be safely roll-backed. (It might be better to drop buffers).
>
> That's obviously a no-go. I think you might be able to address this if
> you accept that the command cannot be run in a transaction (like
> CONCURRENTLY). Then you can first do the catalog changes, change the
> persistence level, and commit.
Of course. The next version reverts persistence change at abort.
Thanks!
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2020-11-13 04:22 ` Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-13 04:22 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hello. Before posting the next version, I'd like to explain what this
patch is.
1. The Issue
Bulk data loading is a long-time taking, I/O consuming task. Many
DBAs want that task is faster, even at the cost of increasing risk of
data-loss. wal_level=minimal is an answer to such a
request. Data-loading onto a table that is created in the current
transaction omits WAL-logging and synced at commit.
However, the optimization doesn't benefit the case where the
data-loading is performed onto existing tables. There are quite a few
cases where data is loaded into tables that already contains a lot of
data. Those cases don't take benefit of the optimization.
Another possible solution for bulk data-loading is UNLOGGED
tables. But when we switch LOGGED/UNLOGGED of a table, all the table
content is copied to a newly created heap, which is costly.
2. Proposed Solutions.
There are two proposed solutions are discussed on this mailing
list. One is wal_level = none (*1), which omits WAL-logging almost at
all. Another is extending the existing optimization to the ALTER TABLE
SET LOGGED/UNLOGGED cases, which is to be discussed in this new
thread.
3. In-place Persistence Change
So the attached is a PoC patch of the "another" solution. When we
want to change table persistence in-place, basically we need to do the
following steps.
(the talbe is exclusively locked)
(1) Flip BM_PERMANENT flag of all shared buffer blocks for the heap.
(2) Create or delete the init fork for existing heap.
(3) Flush all buffers of the relation to file system.
(4) Sync heap files.
(5) Make catalog changes.
4. Transactionality
The 1, 2 and 5 above need to be abort-able. 5 is rolled back by
existing infrastructure, and rolling-back of 1 and 2 are achieved by
piggybacking on the pendingDeletes mechanism.
5. Replication
Furthermore, that changes ought to be replicable to standbys. Catalog
changes are replicated as usual.
On-the-fly creation of the init fork leads to recovery mess. Even
though it is removed at abort, if the server crashed before
transaction end, the file is left alone and corrupts database in the
next recovery. I sought a way to create the init fork in
smgrPendingDelete but that needs relcache and relcache is not
available at that late of commit. Finally, I introduced the fifth fork
kind "INITTMP"(_itmp) only to signal that the init file is not
committed. I don't like that way but it seems working fine...
6. SQL Command
The second file in the patchset adds a syntax that changes persistence
of all tables in a tablespace.
ALTER TABLE ALL IN TABLESPACE <tsp> SET LOGGED/UNLOGGED [ NOWAIT ];
7. Testing
I tried to write TAP test for this, but IPC::Run::harness (or
interactive_psql) doesn't seem to work for me. I'm not sure what
exactly is happening but pty redirection doesn't work.
$in = "ls\n"; $out = ""; run ["/usr/bin/bash"], \$in, \$out; print $out;
works but
$in = "ls\n"; $out = ""; run ["/usr/bin/bash"], '<pty<', \$in, '>pty>', \$out; print $out;
doesn't respond.
The patch is attached.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2020-11-13 06:43 ` [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 2 replies; 103+ messages in thread
From: [email protected] @ 2020-11-13 06:43 UTC (permalink / raw)
To: 'Kyotaro Horiguchi' <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi Horiguchi-san,
Thank you for making a patch so quickly. I've started looking at it.
What makes you think this is a PoC? Documentation and test cases? If there's something you think that doesn't work or are concerned about, can you share it?
Do you know the reason why data copy was done before? And, it may be odd for me to ask this, but I think I saw someone referred to the past discussion that eliminating data copy is difficult due to some processing at commit. I can't find it.
(1)
@@ -168,6 +168,8 @@ extern PGDLLIMPORT int32 *LocalRefCount;
*/
#define BufferGetPage(buffer) ((Page)BufferGetBlock(buffer))
+struct SmgrRelationData;
This declaration is already in the file:
/* forward declared, to avoid having to expose buf_internals.h here */
struct WritebackContext;
/* forward declared, to avoid including smgr.h here */
struct SMgrRelationData;
Regards
Takayuki Tsunakawa
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
@ 2020-11-13 07:15 ` [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: [email protected] @ 2020-11-13 07:15 UTC (permalink / raw)
To: [email protected] <[email protected]>; 'Kyotaro Horiguchi' <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hello, Tsunakawa-San
> Do you know the reason why data copy was done before? And, it may be
> odd for me to ask this, but I think I saw someone referred to the past
> discussion that eliminating data copy is difficult due to some processing at
> commit. I can't find it.
I can share 2 sources why to eliminate the data copy is difficult in hackers thread.
Tom's remark and the context to copy relation's data.
https://www.postgresql.org/message-id/flat/31724.1394163360%40sss.pgh.pa.us#[email protected]...
Amit-San quoted this thread and mentioned that point in another thread.
https://www.postgresql.org/message-id/CAA4eK1%2BHDqS%2B1fhs5Jf9o4ZujQT%3DXBZ6sU0kOuEh2hqQAC%2Bt%3Dw%...
Best,
Takamichi Osumi
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
@ 2020-11-13 08:23 ` Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-13 08:23 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 13 Nov 2020 07:15:41 +0000, "[email protected]" <[email protected]> wrote in
> Hello, Tsunakawa-San
>
Thanks for sharing it!
> > Do you know the reason why data copy was done before? And, it may be
> > odd for me to ask this, but I think I saw someone referred to the past
> > discussion that eliminating data copy is difficult due to some processing at
> > commit. I can't find it.
> I can share 2 sources why to eliminate the data copy is difficult in hackers thread.
>
> Tom's remark and the context to copy relation's data.
> https://www.postgresql.org/message-id/flat/31724.1394163360%40sss.pgh.pa.us#[email protected]...
https://www.postgresql.org/message-id/[email protected]...
> No, not really. The issue is more around what happens if we crash
> part way through. At crash recovery time, the system catalogs are not
> available, because the database isn't consistent yet and, anyway, the
> startup process can't be bound to a database, let alone every database
> that might contain unlogged tables. So the sentinel that's used to
> decide whether to flush the contents of a table or index is the
> presence or absence of an _init fork, which the startup process
> obviously can see just fine. The _init fork also tells us what to
> stick in the relation when we reset it; for a table, we can just reset
> to an empty file, but that's not legal for indexes, so the _init fork
> contains a pre-initialized empty index that we can just copy over.
>
> Now, to make an unlogged table logged, you've got to at some stage
> remove those _init forks. But this is not a transactional operation.
> If you remove the _init forks and then the transaction rolls back,
> you've left the system an inconsistent state. If you postpone the
> removal until commit time, then you have a problem if it fails,
It's true. That are the cause of headache.
> particularly if it works for the first file but fails for the second.
> And if you crash at any point before you've fsync'd the containing
> directory, you have no idea which files will still be on disk after a
> hard reboot.
This is not an issue in this patch *except* the case where init fork
is failed to removed but the following removal of inittmp fork
succeeds. Another idea is adding a "not-yet-committed" property to a
fork. I added a new fork type for easiness of the patch but I could
go that way if that is an issue.
> Amit-San quoted this thread and mentioned that point in another thread.
> https://www.postgresql.org/message-id/CAA4eK1%2BHDqS%2B1fhs5Jf9o4ZujQT%3DXBZ6sU0kOuEh2hqQAC%2Bt%3Dw%...
This sounds like a bit differrent discussion. Making part-of-a-table
UNLOGGED looks far difficult to me.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2020-12-04 07:49 ` [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: [email protected] @ 2020-12-04 07:49 UTC (permalink / raw)
To: 'Kyotaro Horiguchi' <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
From: Kyotaro Horiguchi <[email protected]>
> > No, not really. The issue is more around what happens if we crash
> > part way through. At crash recovery time, the system catalogs are not
> > available, because the database isn't consistent yet and, anyway, the
> > startup process can't be bound to a database, let alone every database
> > that might contain unlogged tables. So the sentinel that's used to
> > decide whether to flush the contents of a table or index is the
> > presence or absence of an _init fork, which the startup process
> > obviously can see just fine. The _init fork also tells us what to
> > stick in the relation when we reset it; for a table, we can just reset
> > to an empty file, but that's not legal for indexes, so the _init fork
> > contains a pre-initialized empty index that we can just copy over.
> >
> > Now, to make an unlogged table logged, you've got to at some stage
> > remove those _init forks. But this is not a transactional operation.
> > If you remove the _init forks and then the transaction rolls back,
> > you've left the system an inconsistent state. If you postpone the
> > removal until commit time, then you have a problem if it fails,
>
> It's true. That are the cause of headache.
...
> The current implement is simple. It's enough to just discard old or
> new relfilenode according to the current transaction ends with commit
> or abort. Tweaking of relfilenode under use leads-in some skews in
> some places. I used pendingDelete mechanism a bit complexified way
> and a violated an abstraction (I think, calling AM-routines from
> storage.c is not good.) and even introduce a new fork kind only to
> mark a init fork as "not committed yet". There might be better way,
> but I haven't find it.
I have no alternative idea yet, too. I agree that we want to avoid them, especially introducing inittmp fork... Anyway, below are the rest of my review comments for 0001. I want to review 0002 when we have decided to go with 0001.
(2)
XLOG_SMGR_UNLINK seems to necessitate modification of the following comments:
[src/include/catalog/storage_xlog.h]
/*
* Declarations for smgr-related XLOG records
*
* Note: we log file creation and truncation here, but logging of deletion
* actions is handled by xact.c, because it is part of transaction commit.
*/
[src/backend/access/transam/README]
3. Deleting a table, which requires an unlink() that could fail.
Our approach here is to WAL-log the operation first, but to treat failure
of the actual unlink() call as a warning rather than error condition.
Again, this can leave an orphan file behind, but that's cheap compared to
the alternatives. Since we can't actually do the unlink() until after
we've committed the DROP TABLE transaction, throwing an error would be out
of the question anyway. (It may be worth noting that the WAL entry about
the file deletion is actually part of the commit record for the dropping
transaction.)
(3)
+/* This is bit-map, not ordianal numbers */
There seems to be no comments using "bit-map". "Flags for ..." can be seen here and there.
(4)
Some wrong spellings:
+ /* we flush this buffer when swithing to PERMANENT */
swithing -> switching
+ * alredy flushed out by RelationCreate(Drop)InitFork called just
alredy -> already
+ * relation content to be WAL-logged to recovery the table.
recovery -> recover
+ * The inittmp fork works as the sentinel to identify that situaton.
situaton -> situation
(5)
+ table_close(classRel, NoLock);
+
+
+
+
}
These empty lines can be deleted.
(6)
+/*
+ * Perform XLogInsert of an XLOG_SMGR_UNLINK record to WAL.
+ */
+void
+log_smgrbufpersistence(const RelFileNode *rnode, bool persistence)
...
+ * Make an XLOG entry reporting the file unlink.
Not unlink but buffer persistence?
(7)
+ /*
+ * index-init fork needs further initialization. ambuildempty shoud do
+ * WAL-log and file sync by itself but otherwise we do that by myself.
+ */
+ if (rel->rd_rel->relkind == RELKIND_INDEX)
+ rel->rd_indam->ambuildempty(rel);
+ else
+ {
+ log_smgrcreate(&rnode, INIT_FORKNUM);
+ smgrimmedsync(srel, INIT_FORKNUM);
+ }
+
+ /*
+ * We have created the init fork. If server crashes before the current
+ * transaction ends the init fork left alone corrupts data while recovery.
+ * The inittmp fork works as the sentinel to identify that situaton.
+ */
+ smgrcreate(srel, INITTMP_FORKNUM, false);
+ log_smgrcreate(&rnode, INITTMP_FORKNUM);
+ smgrimmedsync(srel, INITTMP_FORKNUM);
If the server crashes between these two processings, only the init fork exists. Is it correct to create the inittmp fork first?
(8)
+ if (inxact_created)
+ {
+ SMgrRelation srel = smgropen(rnode, InvalidBackendId);
+ smgrclose(srel);
+ log_smgrunlink(&rnode, INIT_FORKNUM);
+ smgrunlink(srel, INIT_FORKNUM, false);
+ log_smgrunlink(&rnode, INITTMP_FORKNUM);
+ smgrunlink(srel, INITTMP_FORKNUM, false);
+ return;
+ }
smgrclose() should be called just before return.
Isn't it necessary here to revert buffer persistence state change?
(9)
+void
+smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_unlink(reln->smgr_rnode, forknum, isRedo);
+}
Maybe it's better to restore smgrdounlinkfork() that was removed in the older release. That function includes dropping shared buffers, which can clean up the shared buffers that may be cached by this transaction.
(10)
[RelationDropInitFork]
+ /* revert buffer-persistence changes at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_SET_PERSISTENCE;
+ pending->bufpersistence = false;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
bufpersistence should be true.
(11)
+ BlockNumber block = 0;
...
+ DropRelFileNodeBuffers(rbnode, &pending->unlink_forknum, 1,
+ &block);
"block" is unnecessary and 0 can be passed directly.
(12)
- && pending->backend == InvalidBackendId)
+ && pending->backend == InvalidBackendId &&
+ pending->op == PDOP_DELETE)
nrels++;
It's better to put && at the beginning of the line to follow the existing code here.
(13)
+ table_close(rel, lockmode);
lockmode should be NoLock to retain the lock until transaction completion.
(14)
+ ctl.keysize = sizeof(unlogged_relation_entry);
+ ctl.entrysize = sizeof(unlogged_relation_entry);
+ hash = hash_create("unlogged hash", 32, &ctl, HASH_ELEM);
...
+ memset(key.oid, 0, sizeof(key.oid));
+ memcpy(key.oid, de->d_name, oidchars);
+ ent = hash_search(hash, &key, HASH_FIND, NULL);
keysize should be the oid member of the struct.
Regards
Takayuki Tsunakawa
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
@ 2020-12-24 08:02 ` Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-12-24 08:02 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Thanks for the comment! Sorry for the late reply.
At Fri, 4 Dec 2020 07:49:22 +0000, "[email protected]" <[email protected]> wrote in
> From: Kyotaro Horiguchi <[email protected]>
> > > No, not really. The issue is more around what happens if we crash
> > > part way through. At crash recovery time, the system catalogs are not
> > > available, because the database isn't consistent yet and, anyway, the
> > > startup process can't be bound to a database, let alone every database
> > > that might contain unlogged tables. So the sentinel that's used to
> > > decide whether to flush the contents of a table or index is the
> > > presence or absence of an _init fork, which the startup process
> > > obviously can see just fine. The _init fork also tells us what to
> > > stick in the relation when we reset it; for a table, we can just reset
> > > to an empty file, but that's not legal for indexes, so the _init fork
> > > contains a pre-initialized empty index that we can just copy over.
> > >
> > > Now, to make an unlogged table logged, you've got to at some stage
> > > remove those _init forks. But this is not a transactional operation.
> > > If you remove the _init forks and then the transaction rolls back,
> > > you've left the system an inconsistent state. If you postpone the
> > > removal until commit time, then you have a problem if it fails,
> >
> > It's true. That are the cause of headache.
> ...
> > The current implement is simple. It's enough to just discard old or
> > new relfilenode according to the current transaction ends with commit
> > or abort. Tweaking of relfilenode under use leads-in some skews in
> > some places. I used pendingDelete mechanism a bit complexified way
> > and a violated an abstraction (I think, calling AM-routines from
> > storage.c is not good.) and even introduce a new fork kind only to
> > mark a init fork as "not committed yet". There might be better way,
> > but I haven't find it.
>
> I have no alternative idea yet, too. I agree that we want to avoid them, especially introducing inittmp fork... Anyway, below are the rest of my review comments for 0001. I want to review 0002 when we have decided to go with 0001.
>
>
> (2)
> XLOG_SMGR_UNLINK seems to necessitate modification of the following comments:
>
> [src/include/catalog/storage_xlog.h]
> /*
> * Declarations for smgr-related XLOG records
> *
> * Note: we log file creation and truncation here, but logging of deletion
> * actions is handled by xact.c, because it is part of transaction commit.
> */
Sure. Rewrote it.
> [src/backend/access/transam/README]
> 3. Deleting a table, which requires an unlink() that could fail.
>
> Our approach here is to WAL-log the operation first, but to treat failure
> of the actual unlink() call as a warning rather than error condition.
> Again, this can leave an orphan file behind, but that's cheap compared to
> the alternatives. Since we can't actually do the unlink() until after
> we've committed the DROP TABLE transaction, throwing an error would be out
> of the question anyway. (It may be worth noting that the WAL entry about
> the file deletion is actually part of the commit record for the dropping
> transaction.)
Mmm. I didn't touched theDROP TABLE (RelationDropStorage) path, but I
added a brief description about INITTMP fork to the file.
====
The INITTMP fork file
--------------------------------
An INITTMP fork is created when new relation file is created to mark
the relfilenode needs to be cleaned up at recovery time. The file is
removed at transaction end but is left when the process crashes before
the transaction ends. In contrast to 4 above, failure to remove an
INITTMP file will lead to data loss, in which case the server will
shut down.
====
> (3)
> +/* This is bit-map, not ordianal numbers */
>
> There seems to be no comments using "bit-map". "Flags for ..." can be seen here and there.
I revmoed the comment and use (1 << n) notation to show the fact
instead.
> (4)
> Some wrong spellings:
>
> swithing -> switching
> alredy -> already
> recovery -> recover
> situaton -> situation
Oops! Fixed them.
> (5)
> + table_close(classRel, NoLock);
> +
> +
> +
> +
> }
>
> These empty lines can be deleted.
s/can/should/ :p. Fixed.
>
> (6)
> +/*
> + * Perform XLogInsert of an XLOG_SMGR_UNLINK record to WAL.
> + */
> +void
> +log_smgrbufpersistence(const RelFileNode *rnode, bool persistence)
> ...
> + * Make an XLOG entry reporting the file unlink.
>
> Not unlink but buffer persistence?
Silly copy-pasto. Fixed.
> (7)
> + /*
> + * index-init fork needs further initialization. ambuildempty shoud do
> + * WAL-log and file sync by itself but otherwise we do that by myself.
> + */
> + if (rel->rd_rel->relkind == RELKIND_INDEX)
> + rel->rd_indam->ambuildempty(rel);
> + else
> + {
> + log_smgrcreate(&rnode, INIT_FORKNUM);
> + smgrimmedsync(srel, INIT_FORKNUM);
> + }
> +
> + /*
> + * We have created the init fork. If server crashes before the current
> + * transaction ends the init fork left alone corrupts data while recovery.
> + * The inittmp fork works as the sentinel to identify that situaton.
> + */
> + smgrcreate(srel, INITTMP_FORKNUM, false);
> + log_smgrcreate(&rnode, INITTMP_FORKNUM);
> + smgrimmedsync(srel, INITTMP_FORKNUM);
>
> If the server crashes between these two processings, only the init fork exists. Is it correct to create the inittmp fork first?
Right. I change it that way, and did the same with the new code added
to RelationCreateStorage.
> (8)
> + if (inxact_created)
> + {
> + SMgrRelation srel = smgropen(rnode, InvalidBackendId);
> + smgrclose(srel);
> + log_smgrunlink(&rnode, INIT_FORKNUM);
> + smgrunlink(srel, INIT_FORKNUM, false);
> + log_smgrunlink(&rnode, INITTMP_FORKNUM);
> + smgrunlink(srel, INITTMP_FORKNUM, false);
> + return;
> + }
>
> smgrclose() should be called just before return.
> Isn't it necessary here to revert buffer persistence state change?
Mmm. it's a thinko. I was confused with the case of
close/unlink. Fixed all instacnes of the same.
> (9)
> +void
> +smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo)
> +{
> + smgrsw[reln->smgr_which].smgr_unlink(reln->smgr_rnode, forknum, isRedo);
> +}
>
> Maybe it's better to restore smgrdounlinkfork() that was removed in the older release. That function includes dropping shared buffers, which can clean up the shared buffers that may be cached by this transaction.
INITFORK/INITTMP forks cannot be loaded to shared buffer so it's no
use to drop buffers. I added a comment like that.
| /*
| * INIT/INITTMP forks never be loaded to shared buffer so no point in
| * dropping buffers for these files.
| */
| log_smgrunlink(&rnode, INIT_FORKNUM);
I removed DropRelFileNodeBuffers from PDOP_UNLINK_FORK branch in
smgrDoPendingDeletes and added an assertion and a comment instead.
| /* other forks needs to drop buffers */
| Assert(pending->unlink_forknum == INIT_FORKNUM ||
| pending->unlink_forknum == INITTMP_FORKNUM);
|
| log_smgrunlink(&pending->relnode, pending->unlink_forknum);
| smgrunlink(srel, pending->unlink_forknum, false);
> (10)
> [RelationDropInitFork]
> + /* revert buffer-persistence changes at abort */
> + pending = (PendingRelDelete *)
> + MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
> + pending->relnode = rnode;
> + pending->op = PDOP_SET_PERSISTENCE;
> + pending->bufpersistence = false;
> + pending->backend = InvalidBackendId;
> + pending->atCommit = true;
> + pending->nestLevel = GetCurrentTransactionNestLevel();
> + pending->next = pendingDeletes;
> + pendingDeletes = pending;
> +}
>
> bufpersistence should be true.
RelationDropInitFork() chnages the relation persisitence to
"persistent" so it shoud be reverted to "non-persistent (= false)" at
abort. (I agree that the function name is somewhat confusing...)
> (11)
> + BlockNumber block = 0;
> ...
> + DropRelFileNodeBuffers(rbnode, &pending->unlink_forknum, 1,
> + &block);
>
> "block" is unnecessary and 0 can be passed directly.
I removed the entire function call.
But, I don't think you're right here.
| DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
| int nforks, BlockNumber *firstDelBlock)
Doesn't just passing 0 lead to SEGV?
> (12)
> - && pending->backend == InvalidBackendId)
> + && pending->backend == InvalidBackendId &&
> + pending->op == PDOP_DELETE)
> nrels++;
>
> It's better to put && at the beginning of the line to follow the existing code here.
It's terrible.. Fixed.
> (13)
> + table_close(rel, lockmode);
>
> lockmode should be NoLock to retain the lock until transaction completion.
I tried to recall the reason for that, but didn't come up with
anything. Fixed.
> (14)
> + ctl.keysize = sizeof(unlogged_relation_entry);
> + ctl.entrysize = sizeof(unlogged_relation_entry);
> + hash = hash_create("unlogged hash", 32, &ctl, HASH_ELEM);
> ...
> + memset(key.oid, 0, sizeof(key.oid));
> + memcpy(key.oid, de->d_name, oidchars);
> + ent = hash_search(hash, &key, HASH_FIND, NULL);
>
> keysize should be the oid member of the struct.
It's not a problem since the first member is the oid and perhaps it
seems that I thougth to do someting more on that. Now that I don't
recall what is it and in the first place the key should be just Oid in
the context above. Fixed.
The patch is attached to the next message.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2020-12-25 00:12 ` Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-12-25 00:12 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hello.
At Thu, 24 Dec 2020 17:02:20 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> The patch is attached to the next message.
The reason for separating this message is that I modified this so that
it could solve another issue.
There's a complain about orphan files after crash. [1]
1: https://www.postgresql.org/message-id/[email protected]
That is, the case where a relation file is left alone after a server
crash that happened before the end of the transaction that has created
a relation. As I read this, I noticed this feature can solve the
issue with a small change.
This version gets changes in RelationCreateStorage and
smgrDoPendingDeletes.
Previously inittmp fork is created only along with an init fork. This
version creates one always when a relation storage file is created. As
the result ResetUnloggedRelationsInDbspaceDir removes all forks if the
inttmp fork of a logged relations is found. Now that pendingDeletes
can contain multiple entries for the same relation, it has been
modified not to close the same smgr multiple times.
- It might be better to split 0001 into two peaces.
- The function name ResetUnloggedRelationsInDbspaceDir is no longer
represents the function correctly.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-01-08 05:47 ` Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-01-08 05:47 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 25 Dec 2020 09:12:52 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> Hello.
>
> At Thu, 24 Dec 2020 17:02:20 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > The patch is attached to the next message.
>
> The reason for separating this message is that I modified this so that
> it could solve another issue.
>
> There's a complain about orphan files after crash. [1]
>
> 1: https://www.postgresql.org/message-id/[email protected]
>
> That is, the case where a relation file is left alone after a server
> crash that happened before the end of the transaction that has created
> a relation. As I read this, I noticed this feature can solve the
> issue with a small change.
>
> This version gets changes in RelationCreateStorage and
> smgrDoPendingDeletes.
>
> Previously inittmp fork is created only along with an init fork. This
> version creates one always when a relation storage file is created. As
> the result ResetUnloggedRelationsInDbspaceDir removes all forks if the
> inttmp fork of a logged relations is found. Now that pendingDeletes
> can contain multiple entries for the same relation, it has been
> modified not to close the same smgr multiple times.
>
> - It might be better to split 0001 into two peaces.
>
> - The function name ResetUnloggedRelationsInDbspaceDir is no longer
> represents the function correctly.
As pointed by Robert in another thread [1], persisntence of (at least)
GiST index cannot be flipped in-place due to incompatibility of fake
LSNs with real ones.
This version RelationChangePersistence() is changed not to choose
in-place method for indexes other than btree. It seems to be usable
with all kind of indexes other than Gist, but at the mement it applies
only to btrees.
1: https://www.postgresql.org/message-id/[email protected]...
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-01-08 08:52 ` Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-01-08 08:52 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 08 Jan 2021 14:47:05 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> This version RelationChangePersistence() is changed not to choose
> in-place method for indexes other than btree. It seems to be usable
> with all kind of indexes other than Gist, but at the mement it applies
> only to btrees.
>
> 1: https://www.postgresql.org/message-id/[email protected]...
Hmm. This is not wroking correctly. I'll repost after fixint that.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-01-12 09:58 ` Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-01-12 09:58 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 08 Jan 2021 17:52:21 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> At Fri, 08 Jan 2021 14:47:05 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > This version RelationChangePersistence() is changed not to choose
> > in-place method for indexes other than btree. It seems to be usable
> > with all kind of indexes other than Gist, but at the mement it applies
> > only to btrees.
> >
> > 1: https://www.postgresql.org/message-id/[email protected]...
>
> Hmm. This is not wroking correctly. I'll repost after fixint that.
I think I fixed the misbehavior. ResetUnloggedRelationsInDbspaceDir()
handles file operations in the wrong order and with the wrong logic.
It also needed to drop buffers and forget fsync requests.
I thought that the two cases that this patch is expected to fix
(orphan relation files and uncommited init files) can share the same
"cleanup" fork but that is wrong. I had to add one more additional
fork to differentiate the cases of SET UNLOGGED and of creation of
UNLOGGED tables...
The attached is a new version, that seems working correctly but looks
somewhat messy. I'll continue working.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-01-14 08:32 ` Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-01-14 08:32 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Tue, 12 Jan 2021 18:58:08 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> At Fri, 08 Jan 2021 17:52:21 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > At Fri, 08 Jan 2021 14:47:05 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > > This version RelationChangePersistence() is changed not to choose
> > > in-place method for indexes other than btree. It seems to be usable
> > > with all kind of indexes other than Gist, but at the mement it applies
> > > only to btrees.
> > >
> > > 1: https://www.postgresql.org/message-id/[email protected]...
> >
> > Hmm. This is not wroking correctly. I'll repost after fixint that.
>
> I think I fixed the misbehavior. ResetUnloggedRelationsInDbspaceDir()
> handles file operations in the wrong order and with the wrong logic.
> It also needed to drop buffers and forget fsync requests.
>
> I thought that the two cases that this patch is expected to fix
> (orphan relation files and uncommited init files) can share the same
> "cleanup" fork but that is wrong. I had to add one more additional
> fork to differentiate the cases of SET UNLOGGED and of creation of
> UNLOGGED tables...
>
> The attached is a new version, that seems working correctly but looks
> somewhat messy. I'll continue working.
Commit bea449c635 conflicts with this on the change of the definition
of DropRelFileNodeBuffers. The change simplified this patch by a bit:p
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-03-25 05:08 ` Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-03-25 05:08 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
(I'm not sure when the subject was broken..)
At Thu, 14 Jan 2021 17:32:17 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> Commit bea449c635 conflicts with this on the change of the definition
> of DropRelFileNodeBuffers. The change simplified this patch by a bit:p
In this version, I got rid of the "CLEANUP FORK"s, and added a new
system "Smgr marks". The mark files have the name of the
corresponding fork file followed by ".u" (which means Uncommitted.).
"Uncommited"-marked main fork means the same as the CLEANUP2_FORKNUM
and uncommitted-marked init fork means the same as the CLEANUP_FORKNUM
in the previous version.x
I noticed that the previous version of the patch still leaves an
orphan main fork file after "BEGIN; CREATE TABLE x; ROLLBACK; (crash
before checkpoint)" since the "mark" file (or CLEANUP2_FORKNUM) is
revmoed at rollback. In this version the responsibility to remove the
mark files is moved to SyncPostCheckpoint, where main fork files are
actually removed.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-03-25 05:15 ` Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-03-25 05:15 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Thu, 25 Mar 2021 14:08:05 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> (I'm not sure when the subject was broken..)
>
> At Thu, 14 Jan 2021 17:32:17 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > Commit bea449c635 conflicts with this on the change of the definition
> > of DropRelFileNodeBuffers. The change simplified this patch by a bit:p
>
> In this version, I got rid of the "CLEANUP FORK"s, and added a new
> system "Smgr marks". The mark files have the name of the
> corresponding fork file followed by ".u" (which means Uncommitted.).
> "Uncommited"-marked main fork means the same as the CLEANUP2_FORKNUM
> and uncommitted-marked init fork means the same as the CLEANUP_FORKNUM
> in the previous version.x
>
> I noticed that the previous version of the patch still leaves an
> orphan main fork file after "BEGIN; CREATE TABLE x; ROLLBACK; (crash
> before checkpoint)" since the "mark" file (or CLEANUP2_FORKNUM) is
> revmoed at rollback. In this version the responsibility to remove the
> mark files is moved to SyncPostCheckpoint, where main fork files are
> actually removed.
For the record, I noticed that basebackup could be confused by the
mark files but I haven't looked that yet.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-17 09:10 ` Jakub Wartak <[email protected]>
2021-12-17 12:46 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 2 replies; 103+ messages in thread
From: Jakub Wartak @ 2021-12-17 09:10 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
> Kyotaro wrote:
> > In this version, I got rid of the "CLEANUP FORK"s, and added a new
> > system "Smgr marks". The mark files have the name of the
> > corresponding fork file followed by ".u" (which means Uncommitted.).
> > "Uncommited"-marked main fork means the same as the
> CLEANUP2_FORKNUM
> > and uncommitted-marked init fork means the same as the
> CLEANUP_FORKNUM
> > in the previous version.x
> >
> > I noticed that the previous version of the patch still leaves an
> > orphan main fork file after "BEGIN; CREATE TABLE x; ROLLBACK; (crash
> > before checkpoint)" since the "mark" file (or CLEANUP2_FORKNUM) is
> > revmoed at rollback. In this version the responsibility to remove the
> > mark files is moved to SyncPostCheckpoint, where main fork files are
> > actually removed.
>
> For the record, I noticed that basebackup could be confused by the mark files
> but I haven't looked that yet.
>
Good morning Kyotaro,
the patch didn't apply clean (it's from March; some hunks were failing), so I've fixed it and the combined git format-patch is attached. It did conflict with the following:
b0483263dda - Add support for SET ACCESS METHOD in ALTER TABLE
7b565843a94 - Add call to object access hook at the end of table rewrite in ALTER TABLE
9ce346eabf3 - Report progress of startup operations that take a long time.
f10f0ae420 - Replace RelationOpenSmgr() with RelationGetSmgr().
I'm especially worried if I didn't screw up something/forgot something related to the last one (rd->rd_smgr changes), but I'm getting "All 210 tests passed".
Basic demonstration of this patch (with wal_level=minimal):
create unlogged table t6 (id bigint, t text);
-- produces 110GB table, takes ~5mins
insert into t6 select nextval('s1'), repeat('A', 1000) from generate_series(1, 100000000);
alter table t6 set logged;
on baseline SET LOGGED takes: ~7min10s
on patched SET LOGGED takes: 25s
So basically one can - thanks to this patch - use his application (performing classic INSERTs/UPDATEs/DELETEs, so without the need to rewrite to use COPY) and perform literally batch upload and then just switch the tables to LOGGED.
Some more intensive testing also looks good, assuming table prepared to put pressure on WAL:
create unlogged table t_unlogged (id bigint, t text) partition by hash (id);
create unlogged table t_unlogged_h0 partition of t_unlogged FOR VALUES WITH (modulus 4, remainder 0);
[..]
create unlogged table t_unlogged_h3 partition of t_unlogged FOR VALUES WITH (modulus 4, remainder 3);
Workload would still be pretty heavy on LWLock/BufferContent,WALInsert and Lock/extend .
t_logged.sql = insert into t_logged select nextval('s1'), repeat('A', 1000) from generate_series(1, 1000); # according to pg_wal_stats.wal_bytes generates ~1MB of WAL
t_unlogged.sql = insert into t_unlogged select nextval('s1'), repeat('A', 1000) from generate_series(1, 1000); # according to pg_wal_stats.wal_bytes generates ~3kB of WAL
so using: pgbench -f <tabletypetest>.sql -T 30 -P 1 -c 32 -j 3 t
with synchronous_commit =ON(default):
with t_logged.sql: tps = 229 (lat avg = 138ms)
with t_unlogged.sql tps = 283 (lat avg = 112ms) # almost all on LWLock/WALWrite
with synchronous_commit =OFF:
with t_logged.sql: tps = 413 (lat avg = 77ms)
with t_unloged.sql: tps = 782 (lat avg = 40ms)
Afterwards switching the unlogged ~16GB partitions takes 5s per partition.
As the thread didn't get a lot of traction, I've registered it into current commitfest https://commitfest.postgresql.org/36/3461/ with You as the author and in 'Ready for review' state.
I think it behaves as almost finished one and apparently after reading all those discussions that go back over 10years+ time span about this feature, and lot of failed effort towards wal_level=noWAL I think it would be nice to finally start getting some of that of it into the core.
-Jakub Wartak.
Attachments:
[application/octet-stream] v7-0001-In-place-table-persistence-change-with-new-comman.patch (84.3K, ../../AM8PR07MB8248217A3AFEC0DD4D59ED83F6789@AM8PR07MB8248.eurprd07.prod.outlook.com/2-v7-0001-In-place-table-persistence-change-with-new-comman.patch)
download | inline diff:
From 82ea53c317b5c785d7ee91bcdaea43e9ad2c8f77 Mon Sep 17 00:00:00 2001
From: Jakub Wartak <[email protected]>
Date: Thu, 16 Dec 2021 12:03:42 +0000
Subject: [PATCH v7] In-place table persistence change with new command ALTER
TABLE ALL IN TABLESPACE SET LOGGED/UNLOGGED
Even though ALTER TABLE SET LOGGED/UNLOGGED does not require data
rewriting, currently it runs heap rewrite which causes large amount of
file I/O. This patch makes the command run without heap rewrite.
Addition to that, SET LOGGED while wal_level > minimal emits WAL using
XLOG_FPI instead of massive number of HEAP_INSERT's, which should be
smaller.
Also this allows for the cleanup of files left behind in the crash of
the transaction that created it.
ALTER TABLE ALL IN TABLESPACE SET LOGGED/UNLOGGED: To ease invoking
ALTER TABLE SET LOGGED/UNLOGGED, this command changes relation persistence
of all tables in the specified tablespace.
---
src/backend/access/rmgrdesc/smgrdesc.c | 52 ++++
src/backend/access/transam/README | 8 +
src/backend/access/transam/xlog.c | 17 ++
src/backend/catalog/storage.c | 518 +++++++++++++++++++++++++++++++--
src/backend/commands/tablecmds.c | 374 ++++++++++++++++++++++--
src/backend/nodes/copyfuncs.c | 16 +
src/backend/nodes/equalfuncs.c | 15 +
src/backend/parser/gram.y | 20 ++
src/backend/replication/basebackup.c | 3 +-
src/backend/storage/buffer/bufmgr.c | 88 ++++++
src/backend/storage/file/fd.c | 4 +-
src/backend/storage/file/reinit.c | 318 ++++++++++++++------
src/backend/storage/smgr/md.c | 92 +++++-
src/backend/storage/smgr/smgr.c | 32 ++
src/backend/storage/sync/sync.c | 20 +-
src/backend/tcop/utility.c | 11 +
src/bin/pg_rewind/parsexlog.c | 24 ++
src/common/relpath.c | 47 +--
src/include/catalog/storage.h | 2 +
src/include/catalog/storage_xlog.h | 42 ++-
src/include/commands/tablecmds.h | 2 +
src/include/common/relpath.h | 9 +-
src/include/nodes/nodes.h | 1 +
src/include/nodes/parsenodes.h | 9 +
src/include/storage/bufmgr.h | 2 +
src/include/storage/fd.h | 1 +
src/include/storage/md.h | 8 +-
src/include/storage/reinit.h | 10 +-
src/include/storage/smgr.h | 17 ++
29 files changed, 1583 insertions(+), 179 deletions(-)
diff --git a/src/backend/access/rmgrdesc/smgrdesc.c b/src/backend/access/rmgrdesc/smgrdesc.c
index 7755553d57..d251f22207 100644
--- a/src/backend/access/rmgrdesc/smgrdesc.c
+++ b/src/backend/access/rmgrdesc/smgrdesc.c
@@ -40,6 +40,49 @@ smgr_desc(StringInfo buf, XLogReaderState *record)
xlrec->blkno, xlrec->flags);
pfree(path);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) rec;
+ char *path = relpathperm(xlrec->rnode, xlrec->forkNum);
+
+ appendStringInfoString(buf, path);
+ pfree(path);
+ }
+ else if (info == XLOG_SMGR_MARK)
+ {
+ xl_smgr_mark *xlrec = (xl_smgr_mark *) rec;
+ char *path = GetRelationPath(xlrec->rnode.dbNode,
+ xlrec->rnode.spcNode,
+ xlrec->rnode.relNode,
+ InvalidBackendId,
+ xlrec->forkNum, xlrec->mark);
+ char *action;
+
+ switch (xlrec->action)
+ {
+ case XLOG_SMGR_MARK_CREATE:
+ action = "CREATE";
+ break;
+ case XLOG_SMGR_MARK_UNLINK:
+ action = "DELETE";
+ break;
+ default:
+ action = "<unknown action>";
+ break;
+ }
+
+ appendStringInfo(buf, "%s %s", action, path);
+ pfree(path);
+ }
+ else if (info == XLOG_SMGR_BUFPERSISTENCE)
+ {
+ xl_smgr_bufpersistence *xlrec = (xl_smgr_bufpersistence *) rec;
+ char *path = relpathperm(xlrec->rnode, MAIN_FORKNUM);
+
+ appendStringInfoString(buf, path);
+ appendStringInfo(buf, " persistence %d", xlrec->persistence);
+ pfree(path);
+ }
}
const char *
@@ -55,6 +98,15 @@ smgr_identify(uint8 info)
case XLOG_SMGR_TRUNCATE:
id = "TRUNCATE";
break;
+ case XLOG_SMGR_UNLINK:
+ id = "UNLINK";
+ break;
+ case XLOG_SMGR_MARK:
+ id = "MARK";
+ break;
+ case XLOG_SMGR_BUFPERSISTENCE:
+ id = "BUFPERSISTENCE";
+ break;
}
return id;
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README
index 1edc8180c1..7cf77e4a02 100644
--- a/src/backend/access/transam/README
+++ b/src/backend/access/transam/README
@@ -724,6 +724,14 @@ we must panic and abort recovery. The DBA will have to manually clean up
then restart recovery. This is part of the reason for not writing a WAL
entry until we've successfully done the original action.
+The Smgr MARK files
+--------------------------------
+
+A smgr mark files is created when a new relation file is created to
+mark the relfilenode needs to be cleaned up at recovery time. In
+contrast to 4 above, failure to remove smgr mark files will lead to
+data loss, in which case the server will shut down.
+
Skipping WAL for New RelFileNode
--------------------------------
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1e1fbe957f..59f4c2eacf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "catalog/storage.h"
#include "commands/progress.h"
#include "commands/tablespace.h"
#include "common/controldata_utils.h"
@@ -4564,6 +4565,14 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
{
ereport(DEBUG1,
(errmsg_internal("reached end of WAL in pg_wal, entering archive recovery")));
+
+ /* cleanup garbage files left during crash recovery */
+ ResetUnloggedRelations(UNLOGGED_RELATION_DROP_BUFFER |
+ UNLOGGED_RELATION_CLEANUP);
+
+ /* run rollback cleanup if any */
+ smgrDoPendingDeletes(false);
+
InArchiveRecovery = true;
if (StandbyModeRequested)
StandbyMode = true;
@@ -7824,6 +7833,14 @@ StartupXLOG(void)
}
}
+ /* cleanup garbage files left during crash recovery */
+ if (!InArchiveRecovery)
+ ResetUnloggedRelations(UNLOGGED_RELATION_DROP_BUFFER |
+ UNLOGGED_RELATION_CLEANUP);
+
+ /* run rollback cleanup if any */
+ smgrDoPendingDeletes(false);
+
/* Allow resource managers to do any required cleanup. */
for (rmid = 0; rmid <= RM_MAX_ID; rmid++)
{
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index c5ad28d71f..a3e250515c 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -19,6 +19,7 @@
#include "postgres.h"
+#include "access/amapi.h"
#include "access/parallel.h"
#include "access/visibilitymap.h"
#include "access/xact.h"
@@ -27,6 +28,7 @@
#include "access/xlogutils.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
+#include "common/hashfn.h"
#include "miscadmin.h"
#include "storage/freespace.h"
#include "storage/smgr.h"
@@ -57,9 +59,18 @@ int wal_skip_threshold = 2048; /* in kilobytes */
* but I'm being paranoid.
*/
+#define PDOP_DELETE (1 << 0)
+#define PDOP_UNLINK_FORK (1 << 1)
+#define PDOP_UNLINK_MARK (1 << 2)
+#define PDOP_SET_PERSISTENCE (1 << 3)
+
typedef struct PendingRelDelete
{
RelFileNode relnode; /* relation that may need to be deleted */
+ int op; /* operation mask */
+ bool bufpersistence; /* buffer persistence to set */
+ int unlink_forknum; /* forknum to unlink */
+ StorageMarks unlink_mark; /* mark to unlink */
BackendId backend; /* InvalidBackendId if not a temp rel */
bool atCommit; /* T=delete at commit; F=delete at abort */
int nestLevel; /* xact nesting level of request */
@@ -75,6 +86,24 @@ typedef struct PendingRelSync
static PendingRelDelete *pendingDeletes = NULL; /* head of linked list */
HTAB *pendingSyncHash = NULL;
+typedef struct SRelHashEntry
+{
+ SMgrRelation srel;
+ char status; /* for simplehash use */
+} SRelHashEntry;
+
+/* define hashtable for workarea for pending deletes */
+#define SH_PREFIX srelhash
+#define SH_ELEMENT_TYPE SRelHashEntry
+#define SH_KEY_TYPE SMgrRelation
+#define SH_KEY srel
+#define SH_HASH_KEY(tb, key) \
+ hash_bytes((unsigned char *)&key, sizeof(SMgrRelation))
+#define SH_EQUAL(tb, a, b) (memcmp(&a, &b, sizeof(SMgrRelation)) == 0)
+#define SH_SCOPE static inline
+#define SH_DEFINE
+#define SH_DECLARE
+#include "lib/simplehash.h"
/*
* AddPendingSync
@@ -143,22 +172,48 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
return NULL; /* placate compiler */
}
+ /*
+ * We are going to create a new storage file. If server crashes before the
+ * current transaction ends the file needs to be cleaned up but there's no
+ * clue to the orphan files. The SMGR_MARK_UNCOMMITED mark file works as
+ * the signal of that situation.
+ */
srel = smgropen(rnode, backend);
+ log_smgrcreatemark(&rnode, MAIN_FORKNUM, SMGR_MARK_UNCOMMITTED);
+ smgrcreatemark(srel, MAIN_FORKNUM, SMGR_MARK_UNCOMMITTED, false);
smgrcreate(srel, MAIN_FORKNUM, false);
if (needs_wal)
log_smgrcreate(&srel->smgr_rnode.node, MAIN_FORKNUM);
- /* Add the relation to the list of stuff to delete at abort */
+ /*
+ * Add the relation to the list of stuff to delete at abort. We don't
+ * remove the mark file at commit. It needs to persiste until the main fork
+ * file is actually deleted. See SyncPostCheckpoint.
+ */
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rnode;
+ pending->op = PDOP_DELETE;
pending->backend = backend;
pending->atCommit = false; /* delete if abort */
pending->nestLevel = GetCurrentTransactionNestLevel();
pending->next = pendingDeletes;
pendingDeletes = pending;
+ /* drop cleanup fork at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_MARK;
+ pending->unlink_forknum = MAIN_FORKNUM;
+ pending->unlink_mark = SMGR_MARK_UNCOMMITTED;
+ pending->backend = backend;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
{
Assert(backend == InvalidBackendId);
@@ -169,6 +224,207 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
}
/*
+ * RelationCreateInitFork
+ * Create physical storage for the init fork of a relation.
+ *
+ * Create the init fork for the relation.
+ *
+ * This function is transactional. The creation is WAL-logged, and if the
+ * transaction aborts later on, the init fork will be removed.
+ */
+void
+RelationCreateInitFork(Relation rel)
+{
+ RelFileNode rnode = rel->rd_node;
+ PendingRelDelete *pending;
+ SMgrRelation srel;
+ PendingRelDelete *prev;
+ PendingRelDelete *next;
+ bool create = true;
+
+ /* switch buffer persistence */
+ SetRelationBuffersPersistence(RelationGetSmgr(rel), false, false);
+
+ /*
+ * If we have entries for init-fork operation of this relation, that means
+ * that we have already registered pending delete entries to drop
+ * preexisting init fork since before the current transaction started. This
+ * function reverts that change just by removing the entries.
+ */
+ prev = NULL;
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ (pending->op & PDOP_DELETE) == 0 &&
+ (pending->unlink_forknum == INIT_FORKNUM ||
+ (pending->op & PDOP_SET_PERSISTENCE) != 0))
+ {
+ if (prev)
+ prev->next = next;
+ else
+ pendingDeletes = next;
+ pfree(pending);
+
+ create = false;
+ }
+ else
+ {
+ /* unrelated entry, don't touch it */
+ prev = pending;
+ }
+ }
+
+ if (!create)
+ return;
+
+ /*
+ * We are going to create the init fork. If server crashes before the
+ * current transaction ends the init fork left alone corrupts data while
+ * recovery. The cleanup fork works as the sentinel to identify that
+ * situation.
+ */
+ srel = smgropen(rnode, InvalidBackendId);
+ log_smgrcreatemark(&rnode, INIT_FORKNUM, SMGR_MARK_UNCOMMITTED);
+ smgrcreatemark(srel, INIT_FORKNUM, SMGR_MARK_UNCOMMITTED, false);
+
+ /* We don't have existing init fork, create it. */
+ smgrcreate(srel, INIT_FORKNUM, false);
+
+ /*
+ * index-init fork needs further initialization. ambuildempty shoud do
+ * WAL-log and file sync by itself but otherwise we do that by ourselves.
+ */
+ if (rel->rd_rel->relkind == RELKIND_INDEX)
+ rel->rd_indam->ambuildempty(rel);
+ else
+ {
+ log_smgrcreate(&rnode, INIT_FORKNUM);
+ smgrimmedsync(srel, INIT_FORKNUM);
+ }
+
+ /* drop the init fork, mark file and revert persistence at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK | PDOP_UNLINK_MARK | PDOP_SET_PERSISTENCE;
+ pending->unlink_forknum = INIT_FORKNUM;
+ pending->unlink_mark = SMGR_MARK_UNCOMMITTED;
+ pending->bufpersistence = true;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
+ /* drop mark file at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_MARK;
+ pending->unlink_forknum = INIT_FORKNUM;
+ pending->unlink_mark = SMGR_MARK_UNCOMMITTED;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
+/*
+ * RelationDropInitFork
+ * Delete physical storage for the init fork of a relation.
+ *
+ * Register pending-delete of the init fork. The real deletion is performed by
+ * smgrDoPendingDeletes at commit.
+ *
+ * This function is transactional. If the transaction aborts later on, the
+ * deletion doesn't happen.
+ */
+void
+RelationDropInitFork(Relation rel)
+{
+ RelFileNode rnode = rel->rd_node;
+ PendingRelDelete *pending;
+ PendingRelDelete *prev;
+ PendingRelDelete *next;
+ bool inxact_created = false;
+
+ /* switch buffer persistence */
+ SetRelationBuffersPersistence(RelationGetSmgr(rel), true, false);
+
+ /*
+ * If we have entries for init-fork operations of this relation, that means
+ * that we have created the init fork in the current transaction. We
+ * remove the init fork and mark file immediately in that case. Otherwise
+ * just reister pending-delete for the existing init fork.
+ */
+ prev = NULL;
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ (pending->op & PDOP_DELETE) == 0 &&
+ (pending->unlink_forknum == INIT_FORKNUM ||
+ (pending->op & PDOP_SET_PERSISTENCE) != 0))
+ {
+ /* unlink list entry */
+ if (prev)
+ prev->next = next;
+ else
+ pendingDeletes = next;
+ pfree(pending);
+
+ inxact_created = true;
+ }
+ else
+ {
+ /* unrelated entry, don't touch it */
+ prev = pending;
+ }
+ }
+
+ if (inxact_created)
+ {
+ SMgrRelation srel = smgropen(rnode, InvalidBackendId);
+
+ /*
+ * INIT forks never be loaded to shared buffer so no point in dropping
+ * buffers for such files.
+ */
+ log_smgrunlinkmark(&rnode, INIT_FORKNUM, SMGR_MARK_UNCOMMITTED);
+ smgrunlinkmark(srel, INIT_FORKNUM, SMGR_MARK_UNCOMMITTED, false);
+ log_smgrunlink(&rnode, INIT_FORKNUM);
+ smgrunlink(srel, INIT_FORKNUM, false);
+ return;
+ }
+
+ /* register drop of this init fork file at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK;
+ pending->unlink_forknum = INIT_FORKNUM;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
+ /* revert buffer-persistence changes at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_SET_PERSISTENCE;
+ pending->bufpersistence = false;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
+/*
* Perform XLogInsert of an XLOG_SMGR_CREATE record to WAL.
*/
void
@@ -188,6 +444,88 @@ log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum)
}
/*
+ * Perform XLogInsert of an XLOG_SMGR_UNLINK record to WAL.
+ */
+void
+log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum)
+{
+ xl_smgr_unlink xlrec;
+
+ /*
+ * Make an XLOG entry reporting the file unlink.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.forkNum = forkNum;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_UNLINK | XLR_SPECIAL_REL_UPDATE);
+}
+
+/*
+ * Perform XLogInsert of an XLOG_SMGR_CREATEMARK record to WAL.
+ */
+void
+log_smgrcreatemark(const RelFileNode *rnode, ForkNumber forkNum,
+ StorageMarks mark)
+{
+ xl_smgr_mark xlrec;
+
+ /*
+ * Make an XLOG entry reporting the file creation.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.forkNum = forkNum;
+ xlrec.mark = mark;
+ xlrec.action = XLOG_SMGR_MARK_CREATE;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_MARK | XLR_SPECIAL_REL_UPDATE);
+}
+
+/*
+ * Perform XLogInsert of an XLOG_SMGR_UNLINKMARK record to WAL.
+ */
+void
+log_smgrunlinkmark(const RelFileNode *rnode, ForkNumber forkNum,
+ StorageMarks mark)
+{
+ xl_smgr_mark xlrec;
+
+ /*
+ * Make an XLOG entry reporting the file creation.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.forkNum = forkNum;
+ xlrec.mark = mark;
+ xlrec.action = XLOG_SMGR_MARK_UNLINK;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_MARK | XLR_SPECIAL_REL_UPDATE);
+}
+
+/*
+ * Perform XLogInsert of an XLOG_SMGR_BUFPERSISTENCE record to WAL.
+ */
+void
+log_smgrbufpersistence(const RelFileNode *rnode, bool persistence)
+{
+ xl_smgr_bufpersistence xlrec;
+
+ /*
+ * Make an XLOG entry reporting the change of buffer persistence.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.persistence = persistence;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_BUFPERSISTENCE | XLR_SPECIAL_REL_UPDATE);
+}
+
+/*
* RelationDropStorage
* Schedule unlinking of physical storage at transaction commit.
*/
@@ -200,6 +538,7 @@ RelationDropStorage(Relation rel)
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rel->rd_node;
+ pending->op = PDOP_DELETE;
pending->backend = rel->rd_backend;
pending->atCommit = true; /* delete if commit */
pending->nestLevel = GetCurrentTransactionNestLevel();
@@ -618,59 +957,104 @@ smgrDoPendingDeletes(bool isCommit)
int nrels = 0,
maxrels = 0;
SMgrRelation *srels = NULL;
+ srelhash_hash *close_srels = NULL;
+ bool found;
prev = NULL;
for (pending = pendingDeletes; pending != NULL; pending = next)
{
+ SMgrRelation srel;
+
next = pending->next;
if (pending->nestLevel < nestLevel)
{
/* outer-level entries should not be processed yet */
prev = pending;
+ continue;
}
+
+ /* unlink list entry first, so we don't retry on failure */
+ if (prev)
+ prev->next = next;
else
+ pendingDeletes = next;
+
+ if (pending->atCommit != isCommit)
{
- /* unlink list entry first, so we don't retry on failure */
- if (prev)
- prev->next = next;
- else
- pendingDeletes = next;
- /* do deletion if called for */
- if (pending->atCommit == isCommit)
- {
- SMgrRelation srel;
+ /* must explicitly free the list entry */
+ pfree(pending);
+ /* prev does not change */
+ continue;
+ }
- srel = smgropen(pending->relnode, pending->backend);
+ if (close_srels == NULL)
+ close_srels = srelhash_create(CurrentMemoryContext, 32, NULL);
- /* allocate the initial array, or extend it, if needed */
- if (maxrels == 0)
- {
- maxrels = 8;
- srels = palloc(sizeof(SMgrRelation) * maxrels);
- }
- else if (maxrels <= nrels)
- {
- maxrels *= 2;
- srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
- }
+ srel = smgropen(pending->relnode, pending->backend);
+
+ /* Uniquify the smgr relations */
+ srelhash_insert(close_srels, srel, &found);
- srels[nrels++] = srel;
+ if (pending->op & PDOP_DELETE)
+ {
+ /* allocate the initial array, or extend it, if needed */
+ if (maxrels == 0)
+ {
+ maxrels = 8;
+ srels = palloc(sizeof(SMgrRelation) * maxrels);
}
- /* must explicitly free the list entry */
- pfree(pending);
- /* prev does not change */
+ else if (maxrels <= nrels)
+ {
+ maxrels *= 2;
+ srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
+ }
+
+ srels[nrels++] = srel;
}
+
+ if (pending->op & PDOP_UNLINK_FORK)
+ {
+ /* other forks needs to drop buffers */
+ Assert(pending->unlink_forknum == INIT_FORKNUM);
+
+ /* Don't emit wal while recovery. */
+ if (!InRecovery)
+ log_smgrunlink(&pending->relnode, pending->unlink_forknum);
+ smgrunlink(srel, pending->unlink_forknum, false);
+ }
+
+ if (pending->op & PDOP_UNLINK_MARK)
+ {
+ if (!InRecovery)
+ log_smgrunlinkmark(&pending->relnode,
+ pending->unlink_forknum,
+ pending->unlink_mark);
+ smgrunlinkmark(srel, pending->unlink_forknum,
+ pending->unlink_mark, InRecovery);
+ }
+
+ if (pending->op & PDOP_SET_PERSISTENCE)
+ SetRelationBuffersPersistence(srel, pending->bufpersistence,
+ InRecovery);
}
if (nrels > 0)
{
smgrdounlinkall(srels, nrels, false);
-
- for (int i = 0; i < nrels; i++)
- smgrclose(srels[i]);
-
pfree(srels);
}
+
+ if (close_srels)
+ {
+ srelhash_iterator i;
+ SRelHashEntry *ent;
+
+ /* close smgr relatoins */
+ srelhash_start_iterate(close_srels, &i);
+ while ((ent = srelhash_iterate(close_srels, &i)) != NULL)
+ smgrclose(ent->srel);
+ srelhash_destroy(close_srels);
+ }
}
/*
@@ -840,7 +1224,8 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
for (pending = pendingDeletes; pending != NULL; pending = pending->next)
{
if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
- && pending->backend == InvalidBackendId)
+ && pending->backend == InvalidBackendId
+ && pending->op == PDOP_DELETE)
nrels++;
}
if (nrels == 0)
@@ -853,7 +1238,8 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
for (pending = pendingDeletes; pending != NULL; pending = pending->next)
{
if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
- && pending->backend == InvalidBackendId)
+ && pending->backend == InvalidBackendId &&
+ pending->op == PDOP_DELETE)
{
*rptr = pending->relnode;
rptr++;
@@ -933,6 +1319,15 @@ smgr_redo(XLogReaderState *record)
reln = smgropen(xlrec->rnode, InvalidBackendId);
smgrcreate(reln, xlrec->forkNum, true);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) XLogRecGetData(record);
+ SMgrRelation reln;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ smgrunlink(reln, xlrec->forkNum, true);
+ smgrclose(reln);
+ }
else if (info == XLOG_SMGR_TRUNCATE)
{
xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
@@ -1021,6 +1416,65 @@ smgr_redo(XLogReaderState *record)
FreeFakeRelcacheEntry(rel);
}
+ else if (info == XLOG_SMGR_MARK)
+ {
+ xl_smgr_mark *xlrec = (xl_smgr_mark *) XLogRecGetData(record);
+ SMgrRelation reln;
+ PendingRelDelete *pending;
+ bool created = false;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ switch (xlrec->action)
+ {
+ case XLOG_SMGR_MARK_CREATE:
+ smgrcreatemark(reln, xlrec->forkNum, xlrec->mark, true);
+ created = true;
+ break;
+ case XLOG_SMGR_MARK_UNLINK:
+ smgrunlinkmark(reln, xlrec->forkNum, xlrec->mark, true);
+ break;
+ default:
+ elog(ERROR, "unknown smgr_mark action \"%c\"", xlrec->mark);
+ }
+
+ if (created)
+ {
+ /* revert mark file operation at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = xlrec->rnode;
+ pending->op = PDOP_UNLINK_MARK;
+ pending->unlink_forknum = xlrec->forkNum;
+ pending->unlink_mark = xlrec->mark;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+ }
+ }
+ else if (info == XLOG_SMGR_BUFPERSISTENCE)
+ {
+ xl_smgr_bufpersistence *xlrec =
+ (xl_smgr_bufpersistence *) XLogRecGetData(record);
+ SMgrRelation reln;
+ PendingRelDelete *pending;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ SetRelationBuffersPersistence(reln, xlrec->persistence, true);
+
+ /* revert buffer-persistence changes at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = xlrec->rnode;
+ pending->op = PDOP_SET_PERSISTENCE;
+ pending->bufpersistence = !xlrec->persistence;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+ }
else
elog(PANIC, "smgr_redo: unknown op code %u", info);
}
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index bf42587e38..726a0484f9 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -52,6 +52,7 @@
#include "commands/defrem.h"
#include "commands/event_trigger.h"
#include "commands/policy.h"
+#include "commands/progress.h"
#include "commands/sequence.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -5330,6 +5331,168 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel,
}
/*
+ * RelationChangePersistence: do in-place persistence change of a relation
+ */
+static void
+RelationChangePersistence(AlteredTableInfo *tab, char persistence,
+ LOCKMODE lockmode)
+{
+ Relation rel;
+ Relation classRel;
+ HeapTuple tuple,
+ newtuple;
+ Datum new_val[Natts_pg_class];
+ bool new_null[Natts_pg_class],
+ new_repl[Natts_pg_class];
+ int i;
+ List *relids;
+ ListCell *lc_oid;
+
+ Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE);
+ Assert(lockmode == AccessExclusiveLock);
+
+ /*
+ * Under the following condition, we need to call ATRewriteTable, which
+ * cannot be false in the AT_REWRITE_ALTER_PERSISTENCE case.
+ */
+ Assert(tab->constraints == NULL && tab->partition_constraint == NULL &&
+ tab->newvals == NULL && !tab->verify_new_notnull);
+
+ rel = table_open(tab->relid, lockmode);
+
+ Assert(rel->rd_rel->relpersistence != persistence);
+
+ elog(DEBUG1, "perform im-place persistnce change");
+
+ RelationGetSmgr(rel);
+
+ /*
+ * First we collect all relations that we need to change persistence.
+ */
+
+ /* Collect OIDs of indexes and toast relations */
+ relids = RelationGetIndexList(rel);
+ relids = lcons_oid(rel->rd_id, relids);
+
+ /* Add toast relation if any */
+ if (OidIsValid(rel->rd_rel->reltoastrelid))
+ {
+ List *toastidx;
+ Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode);
+
+ RelationGetSmgr(toastrel);
+ relids = lappend_oid(relids, rel->rd_rel->reltoastrelid);
+ toastidx = RelationGetIndexList(toastrel);
+ relids = list_concat(relids, toastidx);
+ pfree(toastidx);
+ table_close(toastrel, NoLock);
+ }
+
+ table_close(rel, NoLock);
+
+ /* Make changes in storage */
+ classRel = table_open(RelationRelationId, RowExclusiveLock);
+
+ foreach (lc_oid, relids)
+ {
+ Oid reloid = lfirst_oid(lc_oid);
+ Relation r = relation_open(reloid, lockmode);
+
+ /*
+ * Some access methods do not accept in-place persistence change. For
+ * example, GiST uses page LSNs to figure out whether a block has
+ * changed, where UNLOGGED GiST indexes use fake LSNs that are
+ * incompatible with real LSNs used for LOGGED ones.
+ *
+ * XXXX: We don't bother allowing in-place persistence change for index
+ * methods other than btree for now.
+ */
+ if (r->rd_rel->relkind == RELKIND_INDEX &&
+ r->rd_rel->relam != BTREE_AM_OID)
+ {
+ int reindex_flags;
+
+ /* reindex doesn't allow concurrent use of the index */
+ table_close(r, NoLock);
+
+ reindex_flags =
+ REINDEX_REL_SUPPRESS_INDEX_USE |
+ REINDEX_REL_CHECK_CONSTRAINTS;
+
+ /* Set the same persistence with the parent relation. */
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED;
+ else
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
+
+ reindex_index(reloid, reindex_flags, persistence, 0);
+
+ continue;
+ }
+
+ /* Create or drop init fork */
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ RelationCreateInitFork(r);
+ else
+ RelationDropInitFork(r);
+
+ /*
+ * When this relation gets WAL-logged, immediately sync all files but
+ * initfork to establish the initial state on storage. Buffers have
+ * already flushed out by RelationCreate(Drop)InitFork called just
+ * above. Initfork should have been synced as needed.
+ */
+ if (persistence == RELPERSISTENCE_PERMANENT)
+ {
+ for (i = 0 ; i < INIT_FORKNUM ; i++)
+ {
+ if (smgrexists(RelationGetSmgr(r), i))
+ smgrimmedsync(RelationGetSmgr(r), i);
+ }
+ }
+
+ /* Update catalog */
+ tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+ if (!HeapTupleIsValid(tuple))
+ elog(ERROR, "cache lookup failed for relation %u", reloid);
+
+ memset(new_val, 0, sizeof(new_val));
+ memset(new_null, false, sizeof(new_null));
+ memset(new_repl, false, sizeof(new_repl));
+
+ new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence);
+ new_null[Anum_pg_class_relpersistence - 1] = false;
+ new_repl[Anum_pg_class_relpersistence - 1] = true;
+
+ newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
+ new_val, new_null, new_repl);
+
+ CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+
+ /*
+ * While wal_level >= replica, switching to LOGGED requires the
+ * relation content to be WAL-logged to recover the table.
+ */
+ if (persistence == RELPERSISTENCE_PERMANENT && XLogIsNeeded())
+ {
+ ForkNumber fork;
+
+ for (fork = 0; fork < INIT_FORKNUM ; fork++)
+ {
+ if (smgrexists(RelationGetSmgr(r), fork))
+ log_newpage_range(r, fork,
+ 0, smgrnblocks(RelationGetSmgr(r), fork), false);
+ }
+ }
+
+ table_close(r, NoLock);
+ }
+
+ table_close(classRel, NoLock);
+}
+
+/*
* ATRewriteTables: ALTER TABLE phase 3
*/
static void
@@ -5474,32 +5637,55 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode,
* persistence. That wouldn't work for pg_class, but that can't be
* unlogged anyway.
*/
- OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod,
+
+ if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE)
+ RelationChangePersistence(tab, persistence, lockmode);
+ else
+ {
+ /*
+ * Create transient table that will receive the modified data.
+ *
+ * Ensure it is marked correctly as logged or unlogged. We
+ * have to do this here so that buffers for the new relfilenode
+ * will have the right persistence set, and at the same time
+ * ensure that the original filenode's buffers will get read in
+ * with the correct setting (i.e. the original one). Otherwise
+ * a rollback after the rewrite would possibly result with
+ * buffers for the original filenode having the wrong
+ * persistence setting.
+ *
+ * NB: This relies on swap_relation_files() also swapping the
+ * persistence. That wouldn't work for pg_class, but that can't
+ * be unlogged anyway.
+ */
+ OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod,
persistence, lockmode);
+
+ /*
+ * Copy the heap data into the new table with the desired
+ * modifications, and test the current data within the table
+ * against new constraints generated by ALTER TABLE commands.
+ */
+ ATRewriteTable(tab, OIDNewHeap, lockmode);
- /*
- * Copy the heap data into the new table with the desired
- * modifications, and test the current data within the table
- * against new constraints generated by ALTER TABLE commands.
- */
- ATRewriteTable(tab, OIDNewHeap, lockmode);
-
- /*
- * Swap the physical files of the old and new heaps, then rebuild
- * indexes and discard the old heap. We can use RecentXmin for
- * the table's new relfrozenxid because we rewrote all the tuples
- * in ATRewriteTable, so no older Xid remains in the table. Also,
- * we never try to swap toast tables by content, since we have no
- * interest in letting this code work on system catalogs.
- */
- finish_heap_swap(tab->relid, OIDNewHeap,
- false, false, true,
- !OidIsValid(tab->newTableSpace),
- RecentXmin,
- ReadNextMultiXactId(),
- persistence);
-
- InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0);
+ /*
+ * Swap the physical files of the old and new heaps, then rebuild
+ * indexes and discard the old heap. We can use RecentXmin for
+ * the table's new relfrozenxid because we rewrote all the tuples
+ * in ATRewriteTable, so no older Xid remains in the table. Also,
+ * we never try to swap toast tables by content, since we have no
+ * interest in letting this code work on system catalogs.
+ */
+ finish_heap_swap(tab->relid, OIDNewHeap,
+ false, false, true,
+ !OidIsValid(tab->newTableSpace),
+ RecentXmin,
+ ReadNextMultiXactId(),
+ persistence);
+
+ InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0);
+ }
+
}
else
{
@@ -14319,6 +14505,146 @@ AlterTableMoveAll(AlterTableMoveAllStmt *stmt)
return new_tablespaceoid;
}
+/*
+ * Alter Table ALL ... SET LOGGED/UNLOGGED
+ *
+ * Allows a user to change persistence of all objects in a given tablespace in
+ * the current database. Objects can be chosen based on the owner of the
+ * object also, to allow users to change persistene only their objects. The
+ * main permissions handling is done by the lower-level change persistence
+ * function.
+ *
+ * All to-be-modified objects are locked first. If NOWAIT is specified and the
+ * lock can't be acquired then we ereport(ERROR).
+ */
+void
+AlterTableSetLoggedAll(AlterTableSetLoggedAllStmt *stmt)
+{
+ List *relations = NIL;
+ ListCell *l;
+ ScanKeyData key[1];
+ Relation rel;
+ TableScanDesc scan;
+ HeapTuple tuple;
+ Oid tablespaceoid;
+ List *role_oids = roleSpecsToIds(NIL);
+
+ /* Ensure we were not asked to change something we can't */
+ if (stmt->objtype != OBJECT_TABLE)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("only tables can be specified")));
+
+ /* Get the tablespace OID */
+ tablespaceoid = get_tablespace_oid(stmt->tablespacename, false);
+
+ /*
+ * Now that the checks are done, check if we should set either to
+ * InvalidOid because it is our database's default tablespace.
+ */
+ if (tablespaceoid == MyDatabaseTableSpace)
+ tablespaceoid = InvalidOid;
+
+ /*
+ * Walk the list of objects in the tablespace to pick up them. This will
+ * only find objects in our database, of course.
+ */
+ ScanKeyInit(&key[0],
+ Anum_pg_class_reltablespace,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(tablespaceoid));
+
+ rel = table_open(RelationRelationId, AccessShareLock);
+ scan = table_beginscan_catalog(rel, 1, key);
+ while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
+ {
+ Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple);
+ Oid relOid = relForm->oid;
+
+ /*
+ * Do not pick-up objects in pg_catalog as part of this, if an admin
+ * really wishes to do so, they can issue the individual ALTER
+ * commands directly.
+ *
+ * Also, explicitly avoid any shared tables, temp tables, or TOAST
+ * (TOAST will be changed with the main table).
+ */
+ if (IsCatalogNamespace(relForm->relnamespace) ||
+ relForm->relisshared ||
+ isAnyTempNamespace(relForm->relnamespace) ||
+ IsToastNamespace(relForm->relnamespace))
+ continue;
+
+ /* Only pick up the object type requested */
+ if (relForm->relkind != RELKIND_RELATION)
+ continue;
+
+ /* Check if we are only picking-up objects owned by certain roles */
+ if (role_oids != NIL && !list_member_oid(role_oids, relForm->relowner))
+ continue;
+
+ /*
+ * Handle permissions-checking here since we are locking the tables
+ * and also to avoid doing a bunch of work only to fail part-way. Note
+ * that permissions will also be checked by AlterTableInternal().
+ *
+ * Caller must be considered an owner on the table of which we're going
+ * to change persistence.
+ */
+ if (!pg_class_ownercheck(relOid, GetUserId()))
+ aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relOid)),
+ NameStr(relForm->relname));
+
+ if (stmt->nowait &&
+ !ConditionalLockRelationOid(relOid, AccessExclusiveLock))
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_IN_USE),
+ errmsg("aborting because lock on relation \"%s.%s\" is not available",
+ get_namespace_name(relForm->relnamespace),
+ NameStr(relForm->relname))));
+ else
+ LockRelationOid(relOid, AccessExclusiveLock);
+
+ /*
+ * Add to our list of objects of which we're going to change
+ * persistence.
+ */
+ relations = lappend_oid(relations, relOid);
+ }
+
+ table_endscan(scan);
+ table_close(rel, AccessShareLock);
+
+ if (relations == NIL)
+ ereport(NOTICE,
+ (errcode(ERRCODE_NO_DATA_FOUND),
+ errmsg("no matching relations in tablespace \"%s\" found",
+ tablespaceoid == InvalidOid ? "(database default)" :
+ get_tablespace_name(tablespaceoid))));
+
+ /*
+ * Everything is locked, loop through and change persistence of all of the
+ * relations.
+ */
+ foreach(l, relations)
+ {
+ List *cmds = NIL;
+ AlterTableCmd *cmd = makeNode(AlterTableCmd);
+
+ if (stmt->logged)
+ cmd->subtype = AT_SetLogged;
+ else
+ cmd->subtype = AT_SetUnLogged;
+
+ cmds = lappend(cmds, cmd);
+
+ EventTriggerAlterTableStart((Node *) stmt);
+ /* OID is set by AlterTableInternal */
+ AlterTableInternal(lfirst_oid(l), cmds, false);
+ EventTriggerAlterTableEnd();
+ }
+}
+
static void
index_copy_data(Relation rel, RelFileNode newrnode)
{
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index df0b747883..55e38cfe3f 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -4269,6 +4269,19 @@ _copyAlterTableMoveAllStmt(const AlterTableMoveAllStmt *from)
return newnode;
}
+static AlterTableSetLoggedAllStmt *
+_copyAlterTableSetLoggedAllStmt(const AlterTableSetLoggedAllStmt *from)
+{
+ AlterTableSetLoggedAllStmt *newnode = makeNode(AlterTableSetLoggedAllStmt);
+
+ COPY_STRING_FIELD(tablespacename);
+ COPY_SCALAR_FIELD(objtype);
+ COPY_SCALAR_FIELD(logged);
+ COPY_SCALAR_FIELD(nowait);
+
+ return newnode;
+}
+
static CreateExtensionStmt *
_copyCreateExtensionStmt(const CreateExtensionStmt *from)
{
@@ -5622,6 +5635,9 @@ copyObjectImpl(const void *from)
case T_AlterTableMoveAllStmt:
retval = _copyAlterTableMoveAllStmt(from);
break;
+ case T_AlterTableSetLoggedAllStmt:
+ retval = _copyAlterTableSetLoggedAllStmt(from);
+ break;
case T_CreateExtensionStmt:
retval = _copyCreateExtensionStmt(from);
break;
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index cb7ddd463c..a19b7874d7 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -1917,6 +1917,18 @@ _equalAlterTableMoveAllStmt(const AlterTableMoveAllStmt *a,
}
static bool
+_equalAlterTableSetLoggedAllStmt(const AlterTableSetLoggedAllStmt *a,
+ const AlterTableSetLoggedAllStmt *b)
+{
+ COMPARE_STRING_FIELD(tablespacename);
+ COMPARE_SCALAR_FIELD(objtype);
+ COMPARE_SCALAR_FIELD(logged);
+ COMPARE_SCALAR_FIELD(nowait);
+
+ return true;
+}
+
+static bool
_equalCreateExtensionStmt(const CreateExtensionStmt *a, const CreateExtensionStmt *b)
{
COMPARE_STRING_FIELD(extname);
@@ -3625,6 +3637,9 @@ equal(const void *a, const void *b)
case T_AlterTableMoveAllStmt:
retval = _equalAlterTableMoveAllStmt(a, b);
break;
+ case T_AlterTableSetLoggedAllStmt:
+ retval = _equalAlterTableSetLoggedAllStmt(a, b);
+ break;
case T_CreateExtensionStmt:
retval = _equalCreateExtensionStmt(a, b);
break;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 3d4dd43e47..9823d57a54 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -1984,6 +1984,26 @@ AlterTableStmt:
n->nowait = $13;
$$ = (Node *)n;
}
+ | ALTER TABLE ALL IN_P TABLESPACE name SET LOGGED opt_nowait
+ {
+ AlterTableSetLoggedAllStmt *n =
+ makeNode(AlterTableSetLoggedAllStmt);
+ n->tablespacename = $6;
+ n->objtype = OBJECT_TABLE;
+ n->logged = true;
+ n->nowait = $9;
+ $$ = (Node *)n;
+ }
+ | ALTER TABLE ALL IN_P TABLESPACE name SET UNLOGGED opt_nowait
+ {
+ AlterTableSetLoggedAllStmt *n =
+ makeNode(AlterTableSetLoggedAllStmt);
+ n->tablespacename = $6;
+ n->objtype = OBJECT_TABLE;
+ n->logged = false;
+ n->nowait = $9;
+ $$ = (Node *)n;
+ }
| ALTER INDEX qualified_name alter_table_cmds
{
AlterTableStmt *n = makeNode(AlterTableStmt);
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index ec0485705d..45e1a5d817 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -1070,6 +1070,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
bool excludeFound;
ForkNumber relForkNum; /* Type of fork if file is a relation */
int relOidChars; /* Chars in filename that are the rel oid */
+ StorageMarks mark;
/* Skip special stuff */
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
@@ -1120,7 +1121,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
/* Exclude all forks for unlogged tables except the init fork */
if (isDbDir &&
parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
- &relForkNum))
+ &relForkNum, &mark))
{
/* Never exclude init forks */
if (relForkNum != INIT_FORKNUM)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index b4532948d3..dab74bf99a 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -37,6 +37,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "catalog/storage_xlog.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -3155,6 +3156,93 @@ DropRelFileNodeBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum,
}
/* ---------------------------------------------------------------------
+ * SetRelFileNodeBuffersPersistence
+ *
+ * This function changes the persistence of all buffer pages of a relation
+ * then writes all dirty pages of the relation out to disk when switching
+ * to PERMANENT. (or more accurately, out to kernel disk buffers),
+ * ensuring that the kernel has an up-to-date view of the relation.
+ *
+ * Generally, the caller should be holding AccessExclusiveLock on the
+ * target relation to ensure that no other backend is busy dirtying
+ * more blocks of the relation; the effects can't be expected to last
+ * after the lock is released.
+ *
+ * XXX currently it sequentially searches the buffer pool, should be
+ * changed to more clever ways of searching. This routine is not
+ * used in any performance-critical code paths, so it's not worth
+ * adding additional overhead to normal paths to make it go faster;
+ * but see also DropRelFileNodeBuffers.
+ * --------------------------------------------------------------------
+ */
+void
+SetRelationBuffersPersistence(SMgrRelation srel, bool permanent, bool isRedo)
+{
+ int i;
+ RelFileNodeBackend rnode = srel->smgr_rnode;
+
+ Assert (!RelFileNodeBackendIsTemp(rnode));
+
+ if (!isRedo)
+ log_smgrbufpersistence(&srel->smgr_rnode.node, permanent);
+
+ ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+
+ for (i = 0; i < NBuffers; i++)
+ {
+ BufferDesc *bufHdr = GetBufferDescriptor(i);
+ uint32 buf_state;
+
+ if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ continue;
+
+ ReservePrivateRefCountEntry();
+
+ buf_state = LockBufHdr(bufHdr);
+
+ if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ {
+ UnlockBufHdr(bufHdr, buf_state);
+ continue;
+ }
+
+ if (permanent)
+ {
+ /* Init fork is being dropped, drop buffers for it. */
+ if (bufHdr->tag.forkNum == INIT_FORKNUM)
+ {
+ InvalidateBuffer(bufHdr);
+ continue;
+ }
+
+ buf_state |= BM_PERMANENT;
+ pg_atomic_write_u32(&bufHdr->state, buf_state);
+
+ /* we flush this buffer when switching to PERMANENT */
+ if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
+ {
+ PinBuffer_Locked(bufHdr);
+ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr),
+ LW_SHARED);
+ FlushBuffer(bufHdr, srel);
+ LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
+ UnpinBuffer(bufHdr, true);
+ }
+ else
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ else
+ {
+ /* init fork is always BM_PERMANENT. See BufferAlloc */
+ if (bufHdr->tag.forkNum != INIT_FORKNUM)
+ buf_state &= ~BM_PERMANENT;
+
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ }
+}
+
+/* ---------------------------------------------------------------------
* DropRelFileNodesAllBuffers
*
* This function removes from the buffer pool all the pages of all
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 263057841d..8487ae1f02 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -349,8 +349,6 @@ static void pre_sync_fname(const char *fname, bool isdir, int elevel);
static void datadir_fsync_fname(const char *fname, bool isdir, int elevel);
static void unlink_if_exists_fname(const char *fname, bool isdir, int elevel);
-static int fsync_parent_path(const char *fname, int elevel);
-
/*
* pg_fsync --- do fsync with or without writethrough
@@ -3759,7 +3757,7 @@ fsync_fname_ext(const char *fname, bool isdir, bool ignore_perm, int elevel)
* This is aimed at making file operations persistent on disk in case of
* an OS crash or power failure.
*/
-static int
+int
fsync_parent_path(const char *fname, int elevel)
{
char parentpath[MAXPGPATH];
diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c
index 0ae3fb6902..a34aa8e9af 100644
--- a/src/backend/storage/file/reinit.c
+++ b/src/backend/storage/file/reinit.c
@@ -16,29 +16,49 @@
#include <unistd.h>
+#include "access/xlog.h"
+#include "catalog/pg_tablespace_d.h"
#include "common/relpath.h"
#include "postmaster/startup.h"
+#include "storage/bufmgr.h"
#include "storage/copydir.h"
#include "storage/fd.h"
+#include "storage/md.h"
#include "storage/reinit.h"
+#include "storage/smgr.h"
#include "utils/hsearch.h"
#include "utils/memutils.h"
static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
- int op);
+ Oid tspid, int op);
static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
- int op);
+ Oid tspid, Oid dbid, int op);
typedef struct
{
Oid reloid; /* hash key */
-} unlogged_relation_entry;
+ bool has_init; /* has INIT fork */
+ bool dirty_init; /* needs to remove INIT fork */
+ bool dirty_all; /* needs to remove all forks */
+} relfile_entry;
/*
- * Reset unlogged relations from before the last restart.
+ * Clean up and reset relation files from before the last restart.
*
- * If op includes UNLOGGED_RELATION_CLEANUP, we remove all forks of any
- * relation with an "init" fork, except for the "init" fork itself.
+ * If op includes UNLOGGED_RELATION_CLEANUP, we perform different operations
+ * depending on the existence of the "cleanup" forks.
+ *
+ * If SMGR_MARK_UNCOMMITTED mark file for init fork is present, we remove the
+ * init fork along with the mark file.
+ *
+ * If SMGR_MARK_UNCOMMITTED mark file for main fork is present we remove the
+ * whole relation along with the mark file.
+ *
+ * Otherwise, if the "init" fork is found. we remove all forks of any relation
+ * with the "init" fork, except for the "init" fork itself.
+ *
+ * If op includes UNLOGGED_RELATION_DROP_BUFFER, we drop all buffers for all
+ * relations that have the "cleanup" and/or the "init" forks.
*
* If op includes UNLOGGED_RELATION_INIT, we copy the "init" fork to the main
* fork.
@@ -72,7 +92,7 @@ ResetUnloggedRelations(int op)
/*
* First process unlogged files in pg_default ($PGDATA/base)
*/
- ResetUnloggedRelationsInTablespaceDir("base", op);
+ ResetUnloggedRelationsInTablespaceDir("base", DEFAULTTABLESPACE_OID, op);
/*
* Cycle through directories for all non-default tablespaces.
@@ -81,13 +101,19 @@ ResetUnloggedRelations(int op)
while ((spc_de = ReadDir(spc_dir, "pg_tblspc")) != NULL)
{
+ Oid tspid;
+
if (strcmp(spc_de->d_name, ".") == 0 ||
strcmp(spc_de->d_name, "..") == 0)
continue;
snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s",
spc_de->d_name, TABLESPACE_VERSION_DIRECTORY);
- ResetUnloggedRelationsInTablespaceDir(temp_path, op);
+
+ tspid = atooid(spc_de->d_name);
+ Assert(tspid != 0);
+
+ ResetUnloggedRelationsInTablespaceDir(temp_path, tspid, op);
}
FreeDir(spc_dir);
@@ -103,7 +129,8 @@ ResetUnloggedRelations(int op)
* Process one tablespace directory for ResetUnloggedRelations
*/
static void
-ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
+ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
+ Oid tspid, int op)
{
DIR *ts_dir;
struct dirent *de;
@@ -130,6 +157,8 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
while ((de = ReadDir(ts_dir, tsdirname)) != NULL)
{
+ Oid dbid;
+
/*
* We're only interested in the per-database directories, which have
* numeric names. Note that this code will also (properly) ignore "."
@@ -148,7 +177,9 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
ereport_startup_progress("resetting unlogged relations (cleanup), elapsed time: %ld.%02d s, current path: %s",
dbspace_path);
- ResetUnloggedRelationsInDbspaceDir(dbspace_path, op);
+ dbid = atooid(de->d_name);
+ Assert(dbid != 0);
+ ResetUnloggedRelationsInDbspaceDir(dbspace_path, tspid, dbid, op);
}
FreeDir(ts_dir);
@@ -158,125 +189,228 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
* Process one per-dbspace directory for ResetUnloggedRelations
*/
static void
-ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
+ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
+ Oid tspid, Oid dbid, int op)
{
DIR *dbspace_dir;
struct dirent *de;
char rm_path[MAXPGPATH * 2];
+ HTAB *hash;
+ HASHCTL ctl;
/* Caller must specify at least one operation. */
- Assert((op & (UNLOGGED_RELATION_CLEANUP | UNLOGGED_RELATION_INIT)) != 0);
+ Assert((op & (UNLOGGED_RELATION_CLEANUP |
+ UNLOGGED_RELATION_DROP_BUFFER |
+ UNLOGGED_RELATION_INIT)) != 0);
/*
* Cleanup is a two-pass operation. First, we go through and identify all
* the files with init forks. Then, we go through again and nuke
* everything with the same OID except the init fork.
*/
- if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
+
+ /*
+ * It's possible that someone could create a ton of unlogged relations
+ * in the same database & tablespace, so we'd better use a hash table
+ * rather than an array or linked list to keep track of which files
+ * need to be reset. Otherwise, this cleanup operation would be
+ * O(n^2).
+ */
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(relfile_entry);
+ hash = hash_create("relfilenode cleanup hash",
+ 32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+ /* Collect INIT fork and mark files in the directory. */
+ dbspace_dir = AllocateDir(dbspacedirname);
+ while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
- HTAB *hash;
- HASHCTL ctl;
+ int oidchars;
+ ForkNumber forkNum;
+ StorageMarks mark;
- /*
- * It's possible that someone could create a ton of unlogged relations
- * in the same database & tablespace, so we'd better use a hash table
- * rather than an array or linked list to keep track of which files
- * need to be reset. Otherwise, this cleanup operation would be
- * O(n^2).
- */
- ctl.keysize = sizeof(Oid);
- ctl.entrysize = sizeof(unlogged_relation_entry);
- ctl.hcxt = CurrentMemoryContext;
- hash = hash_create("unlogged relation OIDs", 32, &ctl,
- HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+ /* Skip anything that doesn't look like a relation data file. */
+ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
+ &forkNum, &mark))
+ continue;
- /* Scan the directory. */
- dbspace_dir = AllocateDir(dbspacedirname);
- while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
+ if (forkNum == INIT_FORKNUM || mark == SMGR_MARK_UNCOMMITTED)
{
- ForkNumber forkNum;
- int oidchars;
- unlogged_relation_entry ent;
-
- /* Skip anything that doesn't look like a relation data file. */
- if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
- &forkNum))
- continue;
-
- /* Also skip it unless this is the init fork. */
- if (forkNum != INIT_FORKNUM)
- continue;
+ Oid key;
+ relfile_entry *ent;
+ bool found;
/*
- * Put the OID portion of the name into the hash table, if it
- * isn't already.
+ * Record the relfilenode information. If it has
+ * SMGR_MARK_UNCOMMITTED mark files, the relfilenode is in dirty
+ * state, where clean up is needed.
*/
- ent.reloid = atooid(de->d_name);
- (void) hash_search(hash, &ent, HASH_ENTER, NULL);
+ key = atooid(de->d_name);
+ ent = hash_search(hash, &key, HASH_ENTER, &found);
+
+ if (!found)
+ {
+ ent->has_init = false;
+ ent->dirty_init = false;
+ ent->dirty_all = false;
+ }
+
+ if (forkNum == INIT_FORKNUM && mark == SMGR_MARK_UNCOMMITTED)
+ ent->dirty_init = true;
+ else if (forkNum == MAIN_FORKNUM && mark == SMGR_MARK_UNCOMMITTED)
+ ent->dirty_all = true;
+ else
+ {
+ Assert(forkNum == INIT_FORKNUM);
+ ent->has_init = true;
+ }
}
+ }
- /* Done with the first pass. */
- FreeDir(dbspace_dir);
+ /* Done with the first pass. */
+ FreeDir(dbspace_dir);
+
+ /* nothing to do if we don't have init nor cleanup forks */
+ if (hash_get_num_entries(hash) < 1)
+ {
+ hash_destroy(hash);
+ return;
+ }
+ if ((op & UNLOGGED_RELATION_DROP_BUFFER) != 0)
+ {
/*
- * If we didn't find any init forks, there's no point in continuing;
- * we can bail out now.
+ * When we come here after recovery, smgr object for this file might
+ * have been created. In that case we need to drop all buffers then the
+ * smgr object before initializing the unlogged relation. This is safe
+ * as far as no other backends have accessed the relation before
+ * starting archive recovery.
*/
- if (hash_get_num_entries(hash) == 0)
+ HASH_SEQ_STATUS status;
+ relfile_entry *ent;
+ SMgrRelation *srels = palloc(sizeof(SMgrRelation) * 8);
+ int maxrels = 8;
+ int nrels = 0;
+ int i;
+
+ Assert(!HotStandbyActive());
+
+ hash_seq_init(&status, hash);
+ while((ent = (relfile_entry *) hash_seq_search(&status)) != NULL)
{
- hash_destroy(hash);
- return;
+ RelFileNodeBackend rel;
+
+ /*
+ * The relation is persistent and stays remain persistent. Don't
+ * drop the buffers for this relation.
+ */
+ if (ent->has_init && ent->dirty_init)
+ continue;
+
+ if (maxrels <= nrels)
+ {
+ maxrels *= 2;
+ srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
+ }
+
+ rel.backend = InvalidBackendId;
+ rel.node.spcNode = tspid;
+ rel.node.dbNode = dbid;
+ rel.node.relNode = ent->reloid;
+
+ srels[nrels++] = smgropen(rel.node, InvalidBackendId);
}
- /*
- * Now, make a second pass and remove anything that matches.
- */
+ DropRelFileNodesAllBuffers(srels, nrels);
+
+ for (i = 0 ; i < nrels ; i++)
+ smgrclose(srels[i]);
+ }
+
+ /*
+ * Now, make a second pass and remove anything that matches.
+ */
+ if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
+ {
dbspace_dir = AllocateDir(dbspacedirname);
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
- ForkNumber forkNum;
- int oidchars;
- unlogged_relation_entry ent;
+ ForkNumber forkNum;
+ StorageMarks mark;
+ int oidchars;
+ Oid key;
+ relfile_entry *ent;
+ RelFileNodeBackend rel;
/* Skip anything that doesn't look like a relation data file. */
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
- &forkNum))
- continue;
-
- /* We never remove the init fork. */
- if (forkNum == INIT_FORKNUM)
+ &forkNum, &mark))
continue;
/*
* See whether the OID portion of the name shows up in the hash
* table. If so, nuke it!
*/
- ent.reloid = atooid(de->d_name);
- if (hash_search(hash, &ent, HASH_FIND, NULL))
+ key = atooid(de->d_name);
+ ent = hash_search(hash, &key, HASH_FIND, NULL);
+
+ if (!ent)
+ continue;
+
+ if (!ent->dirty_all)
{
- snprintf(rm_path, sizeof(rm_path), "%s/%s",
- dbspacedirname, de->d_name);
- if (unlink(rm_path) < 0)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not remove file \"%s\": %m",
- rm_path)));
+ /* clean permanent relations don't need cleanup */
+ if (!ent->has_init)
+ continue;
+
+ if (ent->dirty_init)
+ {
+ /*
+ * The crashed trasaction did SET UNLOGGED. This relation
+ * is restored to a LOGGED relation.
+ */
+ if (forkNum != INIT_FORKNUM)
+ continue;
+ }
else
- elog(DEBUG2, "unlinked file \"%s\"", rm_path);
+ {
+ /*
+ * we don't remove the INIT fork of a non-dirty
+ * relfilenode
+ */
+ if (forkNum == INIT_FORKNUM && mark == SMGR_MARK_NONE)
+ continue;
+ }
}
+
+ /* so, nuke it! */
+ snprintf(rm_path, sizeof(rm_path), "%s/%s",
+ dbspacedirname, de->d_name);
+ if (unlink(rm_path) < 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not remove file \"%s\": %m",
+ rm_path)));
+
+ rel.backend = InvalidBackendId;
+ rel.node.spcNode = tspid;
+ rel.node.dbNode = dbid;
+ rel.node.relNode = atooid(de->d_name);
+
+ ForgetRelationForkSyncRequests(rel, forkNum);
}
/* Cleanup is complete. */
FreeDir(dbspace_dir);
- hash_destroy(hash);
}
+ hash_destroy(hash);
+ hash = NULL;
+
/*
* Initialization happens after cleanup is complete: we copy each init
- * fork file to the corresponding main fork file. Note that if we are
- * asked to do both cleanup and init, we may never get here: if the
- * cleanup code determines that there are no init forks in this dbspace,
- * it will return before we get to this point.
+ * fork file to the corresponding main fork file.
*/
if ((op & UNLOGGED_RELATION_INIT) != 0)
{
@@ -285,6 +419,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
ForkNumber forkNum;
+ StorageMarks mark;
int oidchars;
char oidbuf[OIDCHARS + 1];
char srcpath[MAXPGPATH * 2];
@@ -292,9 +427,11 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
/* Skip anything that doesn't look like a relation data file. */
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
- &forkNum))
+ &forkNum, &mark))
continue;
+ Assert(mark == SMGR_MARK_NONE);
+
/* Also skip it unless this is the init fork. */
if (forkNum != INIT_FORKNUM)
continue;
@@ -328,15 +465,18 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
ForkNumber forkNum;
+ StorageMarks mark;
int oidchars;
char oidbuf[OIDCHARS + 1];
char mainpath[MAXPGPATH];
/* Skip anything that doesn't look like a relation data file. */
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
- &forkNum))
+ &forkNum, &mark))
continue;
+ Assert(mark == SMGR_MARK_NONE);
+
/* Also skip it unless this is the init fork. */
if (forkNum != INIT_FORKNUM)
continue;
@@ -379,7 +519,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
*/
bool
parse_filename_for_nontemp_relation(const char *name, int *oidchars,
- ForkNumber *fork)
+ ForkNumber *fork, StorageMarks *mark)
{
int pos;
@@ -410,11 +550,19 @@ parse_filename_for_nontemp_relation(const char *name, int *oidchars,
for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar)
;
- if (segchar <= 1)
- return false;
- pos += segchar;
+ if (segchar > 1)
+ pos += segchar;
}
+ /* mark file? */
+ if (name[pos] == '.' && name[pos + 1] != 0)
+ {
+ *mark = name[pos + 1];
+ pos += 2;
+ }
+ else
+ *mark = SMGR_MARK_NONE;
+
/* Now we should be at the end. */
if (name[pos] != '\0')
return false;
diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c
index b4bca7eed6..27ca9c1ca2 100644
--- a/src/backend/storage/smgr/md.c
+++ b/src/backend/storage/smgr/md.c
@@ -139,7 +139,8 @@ static MdfdVec *_mdfd_getseg(SMgrRelation reln, ForkNumber forkno,
BlockNumber blkno, bool skipFsync, int behavior);
static BlockNumber _mdnblocks(SMgrRelation reln, ForkNumber forknum,
MdfdVec *seg);
-
+static bool mdmarkexists(SMgrRelation reln, ForkNumber forkNum,
+ StorageMarks mark);
/*
* mdinit() -- Initialize private state for magnetic disk storage manager.
@@ -170,6 +171,80 @@ mdexists(SMgrRelation reln, ForkNumber forkNum)
}
/*
+ * mdcreatemark() -- Create a mark file.
+ *
+ * If isRedo is true, it's okay for the file to exist already.
+ */
+void
+mdcreatemark(SMgrRelation reln, ForkNumber forkNum, StorageMarks mark,
+ bool isRedo)
+{
+ char *path =markpath(reln->smgr_rnode, forkNum, mark);
+ int fd;
+
+ /* See mdcreate for details.. */
+ TablespaceCreateDbspace(reln->smgr_rnode.node.spcNode,
+ reln->smgr_rnode.node.dbNode,
+ isRedo);
+
+ fd = BasicOpenFile(path, O_WRONLY | O_CREAT | O_EXCL);
+ if (fd < 0 && (!isRedo || errno != EEXIST))
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not crete mark file \"%s\": %m", path)));
+
+ pfree(path);
+ pg_fsync(fd);
+ close(fd);
+
+ /*
+ * To guarantee that the creation of the file is persistent, fsync its
+ * parent directory.
+ */
+ fsync_parent_path(path, ERROR);
+}
+
+
+/*
+ * mdunlinkmark() -- Delete the mark file
+ *
+ * If isRedo is true, it's okay for the file being not found.
+ */
+void
+mdunlinkmark(SMgrRelation reln, ForkNumber forkNum, StorageMarks mark,
+ bool isRedo)
+{
+ char *path = markpath(reln->smgr_rnode, forkNum, mark);
+
+ if (!isRedo || mdmarkexists(reln, forkNum, mark))
+ durable_unlink(path, ERROR);
+
+ pfree(path);
+}
+
+/*
+ * mdmarkexists() -- Check if the file exists.
+ */
+static bool
+mdmarkexists(SMgrRelation reln, ForkNumber forkNum, StorageMarks mark)
+{
+ char *path = markpath(reln->smgr_rnode, forkNum, mark);
+ int fd;
+
+ fd = BasicOpenFile(path, O_RDONLY);
+ if (fd < 0 && errno != ENOENT)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not access mark file \"%s\": %m", path)));
+ pfree(path);
+
+ if (fd < 0)
+ return false;
+
+ return true;
+}
+
+/*
* mdcreate() -- Create a new relation on magnetic disk.
*
* If isRedo is true, it's okay for the relation to exist already.
@@ -1026,6 +1101,15 @@ register_forget_request(RelFileNodeBackend rnode, ForkNumber forknum,
}
/*
+ * ForgetRelationForkSyncRequests -- forget any fsyncs and unlinks for a fork
+ */
+void
+ForgetRelationForkSyncRequests(RelFileNodeBackend rnode, ForkNumber forknum)
+{
+ register_forget_request(rnode, forknum, 0);
+}
+
+/*
* ForgetDatabaseSyncRequests -- forget any fsyncs and unlinks for a DB
*/
void
@@ -1378,12 +1462,14 @@ mdsyncfiletag(const FileTag *ftag, char *path)
* Return 0 on success, -1 on failure, with errno set.
*/
int
-mdunlinkfiletag(const FileTag *ftag, char *path)
+mdunlinkfiletag(const FileTag *ftag, char *path, StorageMarks mark)
{
char *p;
/* Compute the path. */
- p = relpathperm(ftag->rnode, MAIN_FORKNUM);
+ p = GetRelationPath(ftag->rnode.dbNode, ftag->rnode.spcNode,
+ ftag->rnode.relNode, InvalidBackendId, MAIN_FORKNUM,
+ mark);
strlcpy(path, p, MAXPGPATH);
pfree(p);
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index 0fcef4994b..110e64b0b2 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -62,6 +62,10 @@ typedef struct f_smgr
void (*smgr_truncate) (SMgrRelation reln, ForkNumber forknum,
BlockNumber nblocks);
void (*smgr_immedsync) (SMgrRelation reln, ForkNumber forknum);
+ void (*smgr_createmark) (SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
+ void (*smgr_unlinkmark) (SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
} f_smgr;
static const f_smgr smgrsw[] = {
@@ -82,6 +86,8 @@ static const f_smgr smgrsw[] = {
.smgr_nblocks = mdnblocks,
.smgr_truncate = mdtruncate,
.smgr_immedsync = mdimmedsync,
+ .smgr_createmark = mdcreatemark,
+ .smgr_unlinkmark = mdunlinkmark,
}
};
@@ -336,6 +342,26 @@ smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo)
}
/*
+ * smgrcreatemark() -- Create a mark file
+ */
+void
+smgrcreatemark(SMgrRelation reln, ForkNumber forknum, StorageMarks mark,
+ bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_createmark(reln, forknum, mark, isRedo);
+}
+
+/*
+ * smgrunlinkmark() -- Delete a mark file
+ */
+void
+smgrunlinkmark(SMgrRelation reln, ForkNumber forknum, StorageMarks mark,
+ bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_unlinkmark(reln, forknum, mark, isRedo);
+}
+
+/*
* smgrdosyncall() -- Immediately sync all forks of all given relations
*
* All forks of all given relations are synced out to the store.
@@ -662,6 +688,12 @@ smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
smgrsw[reln->smgr_which].smgr_immedsync(reln, forknum);
}
+void
+smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_unlink(reln->smgr_rnode, forknum, isRedo);
+}
+
/*
* AtEOXact_SMgr
*
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index d4083e8a56..9563940d45 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -89,7 +89,8 @@ static CycleCtr checkpoint_cycle_ctr = 0;
typedef struct SyncOps
{
int (*sync_syncfiletag) (const FileTag *ftag, char *path);
- int (*sync_unlinkfiletag) (const FileTag *ftag, char *path);
+ int (*sync_unlinkfiletag) (const FileTag *ftag, char *path,
+ StorageMarks mark);
bool (*sync_filetagmatches) (const FileTag *ftag,
const FileTag *candidate);
} SyncOps;
@@ -222,7 +223,8 @@ SyncPostCheckpoint(void)
/* Unlink the file */
if (syncsw[entry->tag.handler].sync_unlinkfiletag(&entry->tag,
- path) < 0)
+ path,
+ SMGR_MARK_NONE) < 0)
{
/*
* There's a race condition, when the database is dropped at the
@@ -236,6 +238,20 @@ SyncPostCheckpoint(void)
(errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", path)));
}
+ else if (syncsw[entry->tag.handler].sync_unlinkfiletag(
+ &entry->tag, path,
+ SMGR_MARK_UNCOMMITTED) < 0)
+ {
+ /*
+ * And we may have SMGR_MARK_UNCOMMITTED file. Remove it if the
+ * fork files has been successfully removed. It's ok if the file
+ * does not exist.
+ */
+ if (errno != ENOENT)
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("could not remove file \"%s\": %m", path)));
+ }
/* Mark the list entry as canceled, just in case */
entry->canceled = true;
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 1fbc387d47..1483f9a475 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -162,6 +162,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree)
case T_AlterTSConfigurationStmt:
case T_AlterTSDictionaryStmt:
case T_AlterTableMoveAllStmt:
+ case T_AlterTableSetLoggedAllStmt:
case T_AlterTableSpaceOptionsStmt:
case T_AlterTableStmt:
case T_AlterTypeStmt:
@@ -1747,6 +1748,12 @@ ProcessUtilitySlow(ParseState *pstate,
commandCollected = true;
break;
+ case T_AlterTableSetLoggedAllStmt:
+ AlterTableSetLoggedAll((AlterTableSetLoggedAllStmt *) parsetree);
+ /* commands are stashed in AlterTableSetLoggedAll */
+ commandCollected = true;
+ break;
+
case T_DropStmt:
ExecDropStmt((DropStmt *) parsetree, isTopLevel);
/* no commands stashed for DROP */
@@ -2669,6 +2676,10 @@ CreateCommandTag(Node *parsetree)
tag = AlterObjectTypeCommandTag(((AlterTableMoveAllStmt *) parsetree)->objtype);
break;
+ case T_AlterTableSetLoggedAllStmt:
+ tag = AlterObjectTypeCommandTag(((AlterTableSetLoggedAllStmt *) parsetree)->objtype);
+ break;
+
case T_AlterTableStmt:
tag = AlterObjectTypeCommandTag(((AlterTableStmt *) parsetree)->objtype);
break;
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 436df54120..dbc0da5da5 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -407,6 +407,30 @@ extractPageInfo(XLogReaderState *record)
* source system.
*/
}
+ else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_UNLINK)
+ {
+ /*
+ * We can safely ignore these. When we compare the sizes later on,
+ * we'll notice that they differ, and copy the missing tail from
+ * source system.
+ */
+ }
+ else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_MARK)
+ {
+ /*
+ * We can safely ignore these. When we compare the sizes later on,
+ * we'll notice that they differ, and copy the missing tail from
+ * source system.
+ */
+ }
+ else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_BUFPERSISTENCE)
+ {
+ /*
+ * We can safely ignore these. When we compare the sizes later on,
+ * we'll notice that they differ, and copy the missing tail from
+ * source system.
+ */
+ }
else if (rmid == RM_XACT_ID &&
((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
(rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED ||
diff --git a/src/common/relpath.c b/src/common/relpath.c
index 1f5c426ec0..67f24890d6 100644
--- a/src/common/relpath.c
+++ b/src/common/relpath.c
@@ -139,9 +139,15 @@ GetDatabasePath(Oid dbNode, Oid spcNode)
*/
char *
GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
- int backendId, ForkNumber forkNumber)
+ int backendId, ForkNumber forkNumber, char mark)
{
char *path;
+ char markstr[10];
+
+ if (mark == 0)
+ markstr[0] = 0;
+ else
+ snprintf(markstr, 10, ".%c", mark);
if (spcNode == GLOBALTABLESPACE_OID)
{
@@ -149,10 +155,10 @@ GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
Assert(dbNode == 0);
Assert(backendId == InvalidBackendId);
if (forkNumber != MAIN_FORKNUM)
- path = psprintf("global/%u_%s",
- relNode, forkNames[forkNumber]);
+ path = psprintf("global/%u_%s%s",
+ relNode, forkNames[forkNumber], markstr);
else
- path = psprintf("global/%u", relNode);
+ path = psprintf("global/%u%s", relNode, markstr);
}
else if (spcNode == DEFAULTTABLESPACE_OID)
{
@@ -160,22 +166,22 @@ GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
if (backendId == InvalidBackendId)
{
if (forkNumber != MAIN_FORKNUM)
- path = psprintf("base/%u/%u_%s",
+ path = psprintf("base/%u/%u_%s%s",
dbNode, relNode,
- forkNames[forkNumber]);
+ forkNames[forkNumber], markstr);
else
- path = psprintf("base/%u/%u",
- dbNode, relNode);
+ path = psprintf("base/%u/%u%s",
+ dbNode, relNode, markstr);
}
else
{
if (forkNumber != MAIN_FORKNUM)
- path = psprintf("base/%u/t%d_%u_%s",
+ path = psprintf("base/%u/t%d_%u_%s%s",
dbNode, backendId, relNode,
- forkNames[forkNumber]);
+ forkNames[forkNumber], markstr);
else
- path = psprintf("base/%u/t%d_%u",
- dbNode, backendId, relNode);
+ path = psprintf("base/%u/t%d_%u%s",
+ dbNode, backendId, relNode, markstr);
}
}
else
@@ -184,27 +190,28 @@ GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
if (backendId == InvalidBackendId)
{
if (forkNumber != MAIN_FORKNUM)
- path = psprintf("pg_tblspc/%u/%s/%u/%u_%s",
+ path = psprintf("pg_tblspc/%u/%s/%u/%u_%s%s",
spcNode, TABLESPACE_VERSION_DIRECTORY,
dbNode, relNode,
- forkNames[forkNumber]);
+ forkNames[forkNumber], markstr);
else
- path = psprintf("pg_tblspc/%u/%s/%u/%u",
+ path = psprintf("pg_tblspc/%u/%s/%u/%u%s",
spcNode, TABLESPACE_VERSION_DIRECTORY,
- dbNode, relNode);
+ dbNode, relNode, markstr);
}
else
{
if (forkNumber != MAIN_FORKNUM)
- path = psprintf("pg_tblspc/%u/%s/%u/t%d_%u_%s",
+ path = psprintf("pg_tblspc/%u/%s/%u/t%d_%u_%s%s",
spcNode, TABLESPACE_VERSION_DIRECTORY,
dbNode, backendId, relNode,
- forkNames[forkNumber]);
+ forkNames[forkNumber], markstr);
else
- path = psprintf("pg_tblspc/%u/%s/%u/t%d_%u",
+ path = psprintf("pg_tblspc/%u/%s/%u/t%d_%u%s",
spcNode, TABLESPACE_VERSION_DIRECTORY,
- dbNode, backendId, relNode);
+ dbNode, backendId, relNode, markstr);
}
}
+
return path;
}
diff --git a/src/include/catalog/storage.h b/src/include/catalog/storage.h
index 0ab32b44e9..382623159c 100644
--- a/src/include/catalog/storage.h
+++ b/src/include/catalog/storage.h
@@ -23,6 +23,8 @@
extern int wal_skip_threshold;
extern SMgrRelation RelationCreateStorage(RelFileNode rnode, char relpersistence);
+extern void RelationCreateInitFork(Relation rel);
+extern void RelationDropInitFork(Relation rel);
extern void RelationDropStorage(Relation rel);
extern void RelationPreserveStorage(RelFileNode rnode, bool atCommit);
extern void RelationPreTruncate(Relation rel);
diff --git a/src/include/catalog/storage_xlog.h b/src/include/catalog/storage_xlog.h
index f0814f1458..12346ed7f6 100644
--- a/src/include/catalog/storage_xlog.h
+++ b/src/include/catalog/storage_xlog.h
@@ -18,17 +18,23 @@
#include "lib/stringinfo.h"
#include "storage/block.h"
#include "storage/relfilenode.h"
+#include "storage/smgr.h"
/*
* Declarations for smgr-related XLOG records
*
- * Note: we log file creation and truncation here, but logging of deletion
- * actions is handled by xact.c, because it is part of transaction commit.
+ * Note: we log file creation, truncation and buffer persistence change here,
+ * but logging of deletion actions is handled mainly by xact.c, because it is
+ * part of transaction commit in most cases. However, there's a case where
+ * init forks are deleted outside control of transaction.
*/
/* XLOG gives us high 4 bits */
#define XLOG_SMGR_CREATE 0x10
#define XLOG_SMGR_TRUNCATE 0x20
+#define XLOG_SMGR_UNLINK 0x30
+#define XLOG_SMGR_MARK 0x40
+#define XLOG_SMGR_BUFPERSISTENCE 0x50
typedef struct xl_smgr_create
{
@@ -36,6 +42,32 @@ typedef struct xl_smgr_create
ForkNumber forkNum;
} xl_smgr_create;
+typedef struct xl_smgr_unlink
+{
+ RelFileNode rnode;
+ ForkNumber forkNum;
+} xl_smgr_unlink;
+
+typedef enum smgr_mark_action
+{
+ XLOG_SMGR_MARK_CREATE = 'c',
+ XLOG_SMGR_MARK_UNLINK = 'u'
+} smgr_mark_action;
+
+typedef struct xl_smgr_mark
+{
+ RelFileNode rnode;
+ ForkNumber forkNum;
+ StorageMarks mark;
+ smgr_mark_action action;
+} xl_smgr_mark;
+
+typedef struct xl_smgr_bufpersistence
+{
+ RelFileNode rnode;
+ bool persistence;
+} xl_smgr_bufpersistence;
+
/* flags for xl_smgr_truncate */
#define SMGR_TRUNCATE_HEAP 0x0001
#define SMGR_TRUNCATE_VM 0x0002
@@ -51,6 +83,12 @@ typedef struct xl_smgr_truncate
} xl_smgr_truncate;
extern void log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum);
+extern void log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum);
+extern void log_smgrcreatemark(const RelFileNode *rnode, ForkNumber forkNum,
+ StorageMarks mark);
+extern void log_smgrunlinkmark(const RelFileNode *rnode, ForkNumber forkNum,
+ StorageMarks mark);
+extern void log_smgrbufpersistence(const RelFileNode *rnode, bool persistence);
extern void smgr_redo(XLogReaderState *record);
extern void smgr_desc(StringInfo buf, XLogReaderState *record);
diff --git a/src/include/commands/tablecmds.h b/src/include/commands/tablecmds.h
index 336549cc5f..714077ff4c 100644
--- a/src/include/commands/tablecmds.h
+++ b/src/include/commands/tablecmds.h
@@ -42,6 +42,8 @@ extern void AlterTableInternal(Oid relid, List *cmds, bool recurse);
extern Oid AlterTableMoveAll(AlterTableMoveAllStmt *stmt);
+extern void AlterTableSetLoggedAll(AlterTableSetLoggedAllStmt *stmt);
+
extern ObjectAddress AlterTableNamespace(AlterObjectSchemaStmt *stmt,
Oid *oldschema);
diff --git a/src/include/common/relpath.h b/src/include/common/relpath.h
index a44be11ca0..106a5cf508 100644
--- a/src/include/common/relpath.h
+++ b/src/include/common/relpath.h
@@ -67,7 +67,7 @@ extern int forkname_chars(const char *str, ForkNumber *fork);
extern char *GetDatabasePath(Oid dbNode, Oid spcNode);
extern char *GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
- int backendId, ForkNumber forkNumber);
+ int backendId, ForkNumber forkNumber, char mark);
/*
* Wrapper macros for GetRelationPath. Beware of multiple
@@ -77,7 +77,7 @@ extern char *GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
/* First argument is a RelFileNode */
#define relpathbackend(rnode, backend, forknum) \
GetRelationPath((rnode).dbNode, (rnode).spcNode, (rnode).relNode, \
- backend, forknum)
+ backend, forknum, 0)
/* First argument is a RelFileNode */
#define relpathperm(rnode, forknum) \
@@ -87,4 +87,9 @@ extern char *GetRelationPath(Oid dbNode, Oid spcNode, Oid relNode,
#define relpath(rnode, forknum) \
relpathbackend((rnode).node, (rnode).backend, forknum)
+/* First argument is a RelFileNodeBackend */
+#define markpath(rnode, forknum, mark) \
+ GetRelationPath((rnode).node.dbNode, (rnode).node.spcNode, \
+ (rnode).node.relNode, \
+ (rnode).backend, forknum, mark)
#endif /* RELPATH_H */
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 7c657c1241..8860b2e548 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -428,6 +428,7 @@ typedef enum NodeTag
T_AlterCollationStmt,
T_CallStmt,
T_AlterStatsStmt,
+ T_AlterTableSetLoggedAllStmt,
/*
* TAGS FOR PARSE TREE NODES (parsenodes.h)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 4c5a8a39bf..c3e1bc66d1 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -2350,6 +2350,15 @@ typedef struct AlterTableMoveAllStmt
bool nowait;
} AlterTableMoveAllStmt;
+typedef struct AlterTableSetLoggedAllStmt
+{
+ NodeTag type;
+ char *tablespacename;
+ ObjectType objtype; /* Object type to move */
+ bool logged;
+ bool nowait;
+} AlterTableSetLoggedAllStmt;
+
/* ----------------------
* Create/Alter Extension Statements
* ----------------------
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index cfce23ecbc..f5a7df87a4 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -206,6 +206,8 @@ extern void FlushRelationsAllBuffers(struct SMgrRelationData **smgrs, int nrels)
extern void FlushDatabaseBuffers(Oid dbid);
extern void DropRelFileNodeBuffers(struct SMgrRelationData *smgr_reln, ForkNumber *forkNum,
int nforks, BlockNumber *firstDelBlock);
+extern void SetRelationBuffersPersistence(struct SMgrRelationData *srel,
+ bool permanent, bool isRedo);
extern void DropRelFileNodesAllBuffers(struct SMgrRelationData **smgr_reln, int nnodes);
extern void DropDatabaseBuffers(Oid dbid);
diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index 34602ae006..2dc0357ad5 100644
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -185,6 +185,7 @@ extern ssize_t pg_pwritev_with_retry(int fd,
extern int pg_truncate(const char *path, off_t length);
extern void fsync_fname(const char *fname, bool isdir);
extern int fsync_fname_ext(const char *fname, bool isdir, bool ignore_perm, int elevel);
+extern int fsync_parent_path(const char *fname, int elevel);
extern int durable_rename(const char *oldfile, const char *newfile, int loglevel);
extern int durable_unlink(const char *fname, int loglevel);
extern int durable_rename_excl(const char *oldfile, const char *newfile, int loglevel);
diff --git a/src/include/storage/md.h b/src/include/storage/md.h
index 752b440864..99620816b5 100644
--- a/src/include/storage/md.h
+++ b/src/include/storage/md.h
@@ -23,6 +23,10 @@
extern void mdinit(void);
extern void mdopen(SMgrRelation reln);
extern void mdclose(SMgrRelation reln, ForkNumber forknum);
+extern void mdcreatemark(SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
+extern void mdunlinkmark(SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
extern void mdcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
extern bool mdexists(SMgrRelation reln, ForkNumber forknum);
extern void mdunlink(RelFileNodeBackend rnode, ForkNumber forknum, bool isRedo);
@@ -41,12 +45,14 @@ extern void mdtruncate(SMgrRelation reln, ForkNumber forknum,
BlockNumber nblocks);
extern void mdimmedsync(SMgrRelation reln, ForkNumber forknum);
+extern void ForgetRelationForkSyncRequests(RelFileNodeBackend rnode,
+ ForkNumber forknum);
extern void ForgetDatabaseSyncRequests(Oid dbid);
extern void DropRelationFiles(RelFileNode *delrels, int ndelrels, bool isRedo);
/* md sync callbacks */
extern int mdsyncfiletag(const FileTag *ftag, char *path);
-extern int mdunlinkfiletag(const FileTag *ftag, char *path);
+extern int mdunlinkfiletag(const FileTag *ftag, char *path, StorageMarks mark);
extern bool mdfiletagmatches(const FileTag *ftag, const FileTag *candidate);
#endif /* MD_H */
diff --git a/src/include/storage/reinit.h b/src/include/storage/reinit.h
index fad1e5c473..e1f97e9b89 100644
--- a/src/include/storage/reinit.h
+++ b/src/include/storage/reinit.h
@@ -16,13 +16,15 @@
#define REINIT_H
#include "common/relpath.h"
-
+#include "storage/smgr.h"
extern void ResetUnloggedRelations(int op);
-extern bool parse_filename_for_nontemp_relation(const char *name,
- int *oidchars, ForkNumber *fork);
+extern bool parse_filename_for_nontemp_relation(const char *name, int *oidchars,
+ ForkNumber *fork,
+ StorageMarks *mark);
#define UNLOGGED_RELATION_CLEANUP 0x0001
-#define UNLOGGED_RELATION_INIT 0x0002
+#define UNLOGGED_RELATION_DROP_BUFFER 0x0002
+#define UNLOGGED_RELATION_INIT 0x0004
#endif /* REINIT_H */
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index a6fbf7b6a6..201ecace8a 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -19,6 +19,18 @@
#include "storage/relfilenode.h"
/*
+ * Storage marks is a file of which existence suggests something about a
+ * file. The name of such files is "<filename>.<mark>", where the mark is one
+ * of the values of StorageMarks. Since ".<digit>" means segment files so don't
+ * use digits for the mark character.
+ */
+typedef enum StorageMarks
+{
+ SMGR_MARK_NONE = 0,
+ SMGR_MARK_UNCOMMITTED = 'u' /* the file is not committed yet */
+} StorageMarks;
+
+/*
* smgr.c maintains a table of SMgrRelation objects, which are essentially
* cached file handles. An SMgrRelation is created (if not already present)
* by smgropen(), and destroyed by smgrclose(). Note that neither of these
@@ -85,7 +97,12 @@ extern void smgrclearowner(SMgrRelation *owner, SMgrRelation reln);
extern void smgrclose(SMgrRelation reln);
extern void smgrcloseall(void);
extern void smgrclosenode(RelFileNodeBackend rnode);
+extern void smgrcreatemark(SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
+extern void smgrunlinkmark(SMgrRelation reln, ForkNumber forknum,
+ StorageMarks mark, bool isRedo);
extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo);
extern void smgrdosyncall(SMgrRelation *rels, int nrels);
extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
--
2.11.0
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-17 12:46 ` Justin Pryzby <[email protected]>
2021-12-17 14:33 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2021-12-17 12:46 UTC (permalink / raw)
To: Jakub Wartak <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
On Fri, Dec 17, 2021 at 09:10:30AM +0000, Jakub Wartak wrote:
> I'm especially worried if I didn't screw up something/forgot something related to the last one (rd->rd_smgr changes), but I'm getting "All 210 tests passed".
> As the thread didn't get a lot of traction, I've registered it into current commitfest https://commitfest.postgresql.org/36/3461/ with You as the author and in 'Ready for review' state.
> I think it behaves as almost finished one and apparently after reading all those discussions that go back over 10years+ time span about this feature, and lot of failed effort towards wal_level=noWAL I think it would be nice to finally start getting some of that of it into the core.
The patch is failing:
http://cfbot.cputube.org/kyotaro-horiguchi.html
https://api.cirrus-ci.com/v1/artifact/task/5564333871595520/regress_diffs/src/bin/pg_upgrade/tmp_che...
I think you ran "make check", but should run something like this:
make check-world -j8 >check-world.log 2>&1 && echo Success
--
Justin
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-17 12:46 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
@ 2021-12-17 14:33 ` Jakub Wartak <[email protected]>
2021-12-17 19:10 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jakub Wartak @ 2021-12-17 14:33 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
> Justin wrote:
> On Fri, Dec 17, 2021 at 09:10:30AM +0000, Jakub Wartak wrote:
> > As the thread didn't get a lot of traction, I've registered it into current
> commitfest
> https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fcommitf
> est.postgresql.org%2F36%2F3461%2F&data=04%7C01%7CJakub.Wartak%
> 40tomtom.com%7Cb815e75090d44e20fd0a08d9c15b45cc%7C374f80267b544a
> 3ab87d328fa26ec10d%7C0%7C0%7C637753420044612362%7CUnknown%7CT
> WFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXV
> CI6Mn0%3D%7C3000&sdata=0BTQSVDnVPu4YpECKXXlBJT5q3Gfgv099SaC
> NuBwiW4%3D&reserved=0 with You as the author and in 'Ready for
> review' state.
>
> The patch is failing:
[..]
> I think you ran "make check", but should run something like this:
> make check-world -j8 >check-world.log 2>&1 && echo Success
Hi Justin,
I've repeated the check-world and it says to me all is ok locally (also with --enable-cassert --enable-debug , at least on Amazon Linux 2) and also installcheck on default params seems to be ok
I don't seem to understand why testfarm reports errors for tests like "path, polygon, rowsecurity" e.g. on Linux/graviton2 and FreeBSD while not on MacOS(?) .
Could someone point to me where to start looking/fixing?
-J.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-17 12:46 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2021-12-17 14:33 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-17 19:10 ` Justin Pryzby <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Justin Pryzby @ 2021-12-17 19:10 UTC (permalink / raw)
To: Jakub Wartak <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
On Fri, Dec 17, 2021 at 02:33:25PM +0000, Jakub Wartak wrote:
> > Justin wrote:
> > On Fri, Dec 17, 2021 at 09:10:30AM +0000, Jakub Wartak wrote:
> > > As the thread didn't get a lot of traction, I've registered it into current
> > commitfest
> > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fcommitf
> > est.postgresql.org%2F36%2F3461%2F&data=04%7C01%7CJakub.Wartak%
> > 40tomtom.com%7Cb815e75090d44e20fd0a08d9c15b45cc%7C374f80267b544a
> > 3ab87d328fa26ec10d%7C0%7C0%7C637753420044612362%7CUnknown%7CT
> > WFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXV
> > CI6Mn0%3D%7C3000&sdata=0BTQSVDnVPu4YpECKXXlBJT5q3Gfgv099SaC
> > NuBwiW4%3D&reserved=0 with You as the author and in 'Ready for
> > review' state.
> >
> > The patch is failing:
> [..]
> > I think you ran "make check", but should run something like this:
> > make check-world -j8 >check-world.log 2>&1 && echo Success
>
> Hi Justin,
>
> I've repeated the check-world and it says to me all is ok locally (also with --enable-cassert --enable-debug , at least on Amazon Linux 2) and also installcheck on default params seems to be ok
> I don't seem to understand why testfarm reports errors for tests like "path, polygon, rowsecurity" e.g. on Linux/graviton2 and FreeBSD while not on MacOS(?) .
> Could someone point to me where to start looking/fixing?
Since it says this, it looks a lot like a memory error like a use-after-free
- like in fsync_parent_path():
CREATE TABLE PATH_TBL (f1 path);
+ERROR: could not open file <....> Pacific": No such file or directory
I see at least this one is still failing, though:
time make -C src/test/recovery check
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-20 06:28 ` Kyotaro Horiguchi <[email protected]>
2021-12-20 07:53 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
1 sibling, 2 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-20 06:28 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hello, Jakub.
At Fri, 17 Dec 2021 09:10:30 +0000, Jakub Wartak <[email protected]> wrote in
> the patch didn't apply clean (it's from March; some hunks were failing), so I've fixed it and the combined git format-patch is attached. It did conflict with the following:
Thanks for looking this. Also thanks for Justin for finding the silly
use-after-free bug. (Now I see the regression test fails and I'm not
sure how come I didn't find this before.)
> b0483263dda - Add support for SET ACCESS METHOD in ALTER TABLE
> 7b565843a94 - Add call to object access hook at the end of table rewrite in ALTER TABLE
> 9ce346eabf3 - Report progress of startup operations that take a long time.
> f10f0ae420 - Replace RelationOpenSmgr() with RelationGetSmgr().
>
> I'm especially worried if I didn't screw up something/forgot something related to the last one (rd->rd_smgr changes), but I'm getting "All 210 tests passed".
About the last one, all rel->rd_smgr acesses need to be repalced with
RelationGetSmgr(). On the other hand we can simply remove
RelationOpenSmgr() calls since the target smgrrelation is guaranteed
to be loaded by RelationGetSmgr().
The fix you made for RelationCreate/DropInitFork is correct and
changes you made would work, but I prefer that the code not being too
permissive for unknown (or unexpected) states.
> Basic demonstration of this patch (with wal_level=minimal):
> create unlogged table t6 (id bigint, t text);
> -- produces 110GB table, takes ~5mins
> insert into t6 select nextval('s1'), repeat('A', 1000) from generate_series(1, 100000000);
> alter table t6 set logged;
> on baseline SET LOGGED takes: ~7min10s
> on patched SET LOGGED takes: 25s
>
> So basically one can - thanks to this patch - use his application (performing classic INSERTs/UPDATEs/DELETEs, so without the need to rewrite to use COPY) and perform literally batch upload and then just switch the tables to LOGGED.
This result is significant. That operation finally requires WAL writes
but I was not sure how much gain FPIs (or bulk WAL logging) gives in
comparison to operational WALs.
> Some more intensive testing also looks good, assuming table prepared to put pressure on WAL:
> create unlogged table t_unlogged (id bigint, t text) partition by hash (id);
> create unlogged table t_unlogged_h0 partition of t_unlogged FOR VALUES WITH (modulus 4, remainder 0);
> [..]
> create unlogged table t_unlogged_h3 partition of t_unlogged FOR VALUES WITH (modulus 4, remainder 3);
>
> Workload would still be pretty heavy on LWLock/BufferContent,WALInsert and Lock/extend .
> t_logged.sql = insert into t_logged select nextval('s1'), repeat('A', 1000) from generate_series(1, 1000); # according to pg_wal_stats.wal_bytes generates ~1MB of WAL
> t_unlogged.sql = insert into t_unlogged select nextval('s1'), repeat('A', 1000) from generate_series(1, 1000); # according to pg_wal_stats.wal_bytes generates ~3kB of WAL
>
> so using: pgbench -f <tabletypetest>.sql -T 30 -P 1 -c 32 -j 3 t
> with synchronous_commit =ON(default):
> with t_logged.sql: tps = 229 (lat avg = 138ms)
> with t_unlogged.sql tps = 283 (lat avg = 112ms) # almost all on LWLock/WALWrite
> with synchronous_commit =OFF:
> with t_logged.sql: tps = 413 (lat avg = 77ms)
> with t_unloged.sql: tps = 782 (lat avg = 40ms)
> Afterwards switching the unlogged ~16GB partitions takes 5s per partition.
>
> As the thread didn't get a lot of traction, I've registered it into current commitfest https://commitfest.postgresql.org/36/3461/ with You as the author and in 'Ready for review' state.
>
> I think it behaves as almost finished one and apparently after reading all those discussions that go back over 10years+ time span about this feature, and lot of failed effort towards wal_level=noWAL I think it would be nice to finally start getting some of that of it into the core.
Thanks for taking the performance benchmark.
I didn't register for some reasons.
1. I'm not sure that we want to have the new mark files.
2. Aside of possible bugs, I'm not confident that the crash-safety of
this patch is actually water-tight. At least we need tests for some
failure cases.
3. As mentioned in transam/README, failure in removing smgr mark files
leads to immediate shut down. I'm not sure this behavior is acceptable.
4. Including the reasons above, this is not fully functionally.
For example, if we execute the following commands on primary,
replica dones't work correctly. (boom!)
=# CREATE UNLOGGED TABLE t (a int);
=# ALTER TABLE t SET LOGGED;
The following fixes are done in the attched v8.
- Rebased. Referring to Jakub and Justin's work, I replaced direct
access to ->rd_smgr with RelationGetSmgr() and removed calls to
RelationOpenSmgr(). I still separate the "ALTER TABLE ALL IN
TABLESPACE SET LOGGED/UNLOGGED" statement part.
- Fixed RelationCreate/DropInitFork's behavior for non-target
relations. (From Jakub's work).
- Fixed wording of some comments.
- As revisited, I found a bug around recovery. If the logged-ness of a
relation gets flipped repeatedly in a transaction, duplicate
pending-delete entries are accumulated during recovery and work in a
wrong way. sgmr_redo now adds up to one entry for a action.
- The issue 4 above is not fixed (yet).
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-20 07:53 ` Kyotaro Horiguchi <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-20 07:53 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Mon, 20 Dec 2021 15:28:23 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
>
> 4. Including the reasons above, this is not fully functionally.
> For example, if we execute the following commands on primary,
> replica dones't work correctly. (boom!)
>
> =# CREATE UNLOGGED TABLE t (a int);
> =# ALTER TABLE t SET LOGGED;
> - The issue 4 above is not fixed (yet).
Not only for the case, RelationChangePersistence needs to send a
truncate record before FPIs. If primary crashes amid of the
operation, the table content will be vanish with the persistence
change. That is the correct behavior.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-20 07:59 ` Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Jakub Wartak @ 2021-12-20 07:59 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi Kyotaro, I'm glad you are still into this
> I didn't register for some reasons.
Right now in v8 there's a typo in ./src/backend/catalog/storage.c :
storage.c: In function 'RelationDropInitFork':
storage.c:385:44: error: expected statement before ')' token
pending->unlink_forknum != INIT_FORKNUM)) <-- here, one ) too much
> 1. I'm not sure that we want to have the new mark files.
I can't help with such design decision, but if there are doubts maybe then add checking return codes around:
a) pg_fsync() and fsync_parent_path() (??) inside mdcreatemark()
b) mdunlinkmark() inside mdunlinkmark()
and PANIC if something goes wrong?
> 2. Aside of possible bugs, I'm not confident that the crash-safety of
> this patch is actually water-tight. At least we need tests for some
> failure cases.
>
> 3. As mentioned in transam/README, failure in removing smgr mark files
> leads to immediate shut down. I'm not sure this behavior is acceptable.
Doesn't it happen for most of the stuff already? There's even data_sync_retry GUC.
> 4. Including the reasons above, this is not fully functionally.
> For example, if we execute the following commands on primary,
> replica dones't work correctly. (boom!)
>
> =# CREATE UNLOGGED TABLE t (a int);
> =# ALTER TABLE t SET LOGGED;
>
>
> The following fixes are done in the attched v8.
>
> - Rebased. Referring to Jakub and Justin's work, I replaced direct
> access to ->rd_smgr with RelationGetSmgr() and removed calls to
> RelationOpenSmgr(). I still separate the "ALTER TABLE ALL IN
> TABLESPACE SET LOGGED/UNLOGGED" statement part.
>
> - Fixed RelationCreate/DropInitFork's behavior for non-target
> relations. (From Jakub's work).
>
> - Fixed wording of some comments.
>
> - As revisited, I found a bug around recovery. If the logged-ness of a
> relation gets flipped repeatedly in a transaction, duplicate
> pending-delete entries are accumulated during recovery and work in a
> wrong way. sgmr_redo now adds up to one entry for a action.
>
> - The issue 4 above is not fixed (yet).
Thanks again, If you have any list of crush tests ideas maybe I'll have some minutes
to try to figure them out. Is there is any goto list of stuff to be checked to add confidence
to this patch (as per point #2) ?
BTW fast feedback regarding that ALTER patch (there were 4 unlogged tables):
# ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
WARNING: unrecognized node type: 349
-J.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-20 08:39 ` Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-20 08:39 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Mon, 20 Dec 2021 07:59:29 +0000, Jakub Wartak <[email protected]> wrote in
> Hi Kyotaro, I'm glad you are still into this
>
> > I didn't register for some reasons.
>
> Right now in v8 there's a typo in ./src/backend/catalog/storage.c :
>
> storage.c: In function 'RelationDropInitFork':
> storage.c:385:44: error: expected statement before ')' token
> pending->unlink_forknum != INIT_FORKNUM)) <-- here, one ) too much
Yeah, I thought that I had removed it. v9 patch I believe is correct.
> > 1. I'm not sure that we want to have the new mark files.
>
> I can't help with such design decision, but if there are doubts maybe then add checking return codes around:
> a) pg_fsync() and fsync_parent_path() (??) inside mdcreatemark()
> b) mdunlinkmark() inside mdunlinkmark()
> and PANIC if something goes wrong?
The point is it is worth the complexity it adds. Since the mark file
can resolve another existing (but I don't recall in detail) issue and
this patchset actually fixes it, it can be said to have a certain
extent of persuasiveness. But that doesn't change the fact that it's
additional complexity.
> > 2. Aside of possible bugs, I'm not confident that the crash-safety of
> > this patch is actually water-tight. At least we need tests for some
> > failure cases.
> >
> > 3. As mentioned in transam/README, failure in removing smgr mark files
> > leads to immediate shut down. I'm not sure this behavior is acceptable.
>
> Doesn't it happen for most of the stuff already? There's even data_sync_retry GUC.
Hmm. Yes, actually it is "as water-tight as possible". I just want
others' eyes on that perspective. CF could be the entry point of
others but I'm a bit hesitent to add a new entry..
> > 4. Including the reasons above, this is not fully functionally.
> > For example, if we execute the following commands on primary,
> > replica dones't work correctly. (boom!)
> >
> > =# CREATE UNLOGGED TABLE t (a int);
> > =# ALTER TABLE t SET LOGGED;
> >
> >
> > The following fixes are done in the attched v8.
> >
> > - Rebased. Referring to Jakub and Justin's work, I replaced direct
> > access to ->rd_smgr with RelationGetSmgr() and removed calls to
> > RelationOpenSmgr(). I still separate the "ALTER TABLE ALL IN
> > TABLESPACE SET LOGGED/UNLOGGED" statement part.
> >
> > - Fixed RelationCreate/DropInitFork's behavior for non-target
> > relations. (From Jakub's work).
> >
> > - Fixed wording of some comments.
> >
> > - As revisited, I found a bug around recovery. If the logged-ness of a
> > relation gets flipped repeatedly in a transaction, duplicate
> > pending-delete entries are accumulated during recovery and work in a
> > wrong way. sgmr_redo now adds up to one entry for a action.
> >
> > - The issue 4 above is not fixed (yet).
>
> Thanks again, If you have any list of crush tests ideas maybe I'll have some minutes
> to try to figure them out. Is there is any goto list of stuff to be checked to add confidence
> to this patch (as per point #2) ?
Just causing a crash (kill -9) after executing problem-prone command
sequence, then seeing recovery works well would sufficient.
For example:
create unlogged table; begin; insert ..; alter table set logged;
<crash>. Recovery works.
"create logged; begin; {alter unlogged; alter logged;} * 1000; alter
logged; commit/abort" doesn't pollute pgdata.
> BTW fast feedback regarding that ALTER patch (there were 4 unlogged tables):
> # ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
> WARNING: unrecognized node type: 349
lol I met a server crash. Will fix. Thanks!
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-20 08:52 ` Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-20 08:52 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Mon, 20 Dec 2021 17:39:27 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> At Mon, 20 Dec 2021 07:59:29 +0000, Jakub Wartak <[email protected]> wrote in
> > BTW fast feedback regarding that ALTER patch (there were 4 unlogged tables):
> > # ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
> > WARNING: unrecognized node type: 349
>
> lol I met a server crash. Will fix. Thanks!
That crash vanished after a recompilation for me and I don't see that
error. On my dev env node# 349 is T_ALterTableSetLoggedAllStmt, which
0002 adds. So perhaps make clean/make all would fix that.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-20 13:38 ` Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jakub Wartak @ 2021-12-20 13:38 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi Kyotaro,
> At Mon, 20 Dec 2021 17:39:27 +0900 (JST), Kyotaro Horiguchi
> <[email protected]> wrote in
> > At Mon, 20 Dec 2021 07:59:29 +0000, Jakub Wartak
> > <[email protected]> wrote in
> > > BTW fast feedback regarding that ALTER patch (there were 4 unlogged
> tables):
> > > # ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
> > > WARNING: unrecognized node type: 349
> >
> > lol I met a server crash. Will fix. Thanks!
>
> That crash vanished after a recompilation for me and I don't see that error. On
> my dev env node# 349 is T_ALterTableSetLoggedAllStmt, which
> 0002 adds. So perhaps make clean/make all would fix that.
The fastest I could - I've repeated the whole cycle about that one with fresh v9 (make clean, configure, make install, fresh initdb) and I've found two problems:
1) check-worlds seems OK but make -C src/test/recovery check shows a couple of failing tests here locally and in https://cirrus-ci.com/task/4699985735319552?logs=test#L807 :
t/009_twophase.pl (Wstat: 256 Tests: 24 Failed: 1)
Failed test: 21
Non-zero exit status: 1
t/014_unlogged_reinit.pl (Wstat: 512 Tests: 12 Failed: 2)
Failed tests: 9-10
Non-zero exit status: 2
t/018_wal_optimize.pl (Wstat: 7424 Tests: 0 Failed: 0)
Non-zero exit status: 29
Parse errors: Bad plan. You planned 38 tests but ran 0.
t/022_crash_temp_files.pl (Wstat: 7424 Tests: 6 Failed: 0)
Non-zero exit status: 29
Parse errors: Bad plan. You planned 9 tests but ran 6.
018 made no sense, I've tried to take a quick look with wal_level=minimal why it is failing , it is mystery to me as the sequence seems to be pretty basic but the outcome is not:
~> cat repro.sql
create tablespace tbs1 location '/tbs1';
CREATE TABLE moved (id int);
INSERT INTO moved VALUES (1);
BEGIN;
ALTER TABLE moved SET TABLESPACE tbs1;
CREATE TABLE originated (id int);
INSERT INTO originated VALUES (1);
CREATE UNIQUE INDEX ON originated(id) TABLESPACE tbs1;
COMMIT;
~> psql -f repro.sql z3; sleep 1; /usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile -m immediate stop
CREATE TABLESPACE
CREATE TABLE
INSERT 0 1
BEGIN
ALTER TABLE
CREATE TABLE
INSERT 0 1
CREATE INDEX
COMMIT
waiting for server to shut down.... done
server stopped
~> /usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile start
waiting for server to start.... done
server started
z3# select * from moved;
ERROR: could not open file "pg_tblspc/32834/PG_15_202112131/32833/32838": No such file or directory
z3=# select * from originated;
ERROR: could not open file "base/32833/32839": No such file or directory
z3=# \dt+
List of relations
Schema | Name | Type | Owner | Persistence | Size | Description
--------+------------+-------+----------+-------------+---------+-------------
public | moved | table | postgres | permanent | 0 bytes |
public | originated | table | postgres | permanent | 0 bytes |
This happens even without placing on tablespace at all {for originated table , but no for moved on}, some major mishap is there (commit should guarantee correctness) or I'm tired and having sloppy fingers.
2) minor one testcase, still something is odd.
drop tablespace tbs1;
create tablespace tbs1 location '/tbs1';
CREATE UNLOGGED TABLE t4 (a int) tablespace tbs1;
CREATE UNLOGGED TABLE t5 (a int) tablespace tbs1;
CREATE UNLOGGED TABLE t6 (a int) tablespace tbs1;
CREATE TABLE t7 (a int) tablespace tbs1;
insert into t7 values (1);
insert into t5 values (1);
insert into t6 values (1);
\dt+
List of relations
Schema | Name | Type | Owner | Persistence | Size | Description
--------+------+-------+----------+-------------+------------+-------------
public | t4 | table | postgres | unlogged | 0 bytes |
public | t5 | table | postgres | unlogged | 8192 bytes |
public | t6 | table | postgres | unlogged | 8192 bytes |
public | t7 | table | postgres | permanent | 8192 bytes |
(4 rows)
ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
==> STILL WARNING: unrecognized node type: 349
\dt+
List of relations
Schema | Name | Type | Owner | Persistence | Size | Description
--------+------+-------+----------+-------------+------------+-------------
public | t4 | table | postgres | permanent | 0 bytes |
public | t5 | table | postgres | permanent | 8192 bytes |
public | t6 | table | postgres | permanent | 8192 bytes |
public | t7 | table | postgres | permanent | 8192 bytes |
So it did rewrite however this warning seems to be unfixed. I've tested on e2c52beecdea152ca680a22ef35c6a7da55aa30f.
-J.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-21 08:13 ` Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-21 08:13 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Ugh! I completely forgot about TAP tests.. Thanks for the testing and
sorry for the bugs.
This is a bit big change so I need a bit of time before posting the
next version.
At Mon, 20 Dec 2021 13:38:35 +0000, Jakub Wartak <[email protected]> wrote in
> 1) check-worlds seems OK but make -C src/test/recovery check shows a couple of failing tests here locally and in https://cirrus-ci.com/task/4699985735319552?logs=test#L807 :
> t/009_twophase.pl (Wstat: 256 Tests: 24 Failed: 1)
> Failed test: 21
> Non-zero exit status: 1
PREPARE TRANSACTION requires uncommited file creation to be
committed. Concretely we need to remove the "mark" files for the
in-transaction created relation file during PREPARE TRANSACTION.
pendingSync is not a parallel mechanism with pendingDeletes so we
cannot move mark deletion to pendingSync.
After all I decided to add a separate list pendingCleanups for pending
non-deletion tasks separately from pendingDeletes and execute it
before insering the commit record. Not only the above but also all of
the following failures vanished by the change.
> t/014_unlogged_reinit.pl (Wstat: 512 Tests: 12 Failed: 2)
> Failed tests: 9-10
> Non-zero exit status: 2
> t/018_wal_optimize.pl (Wstat: 7424 Tests: 0 Failed: 0)
> Non-zero exit status: 29
> Parse errors: Bad plan. You planned 38 tests but ran 0.
> t/022_crash_temp_files.pl (Wstat: 7424 Tests: 6 Failed: 0)
> Non-zero exit status: 29
> Parse errors: Bad plan. You planned 9 tests but ran 6.
> 018 made no sense, I've tried to take a quick look with wal_level=minimal why it is failing , it is mystery to me as the sequence seems to be pretty basic but the outcome is not:
I think this shares the same cause.
> ~> cat repro.sql
> create tablespace tbs1 location '/tbs1';
> CREATE TABLE moved (id int);
> INSERT INTO moved VALUES (1);
> BEGIN;
> ALTER TABLE moved SET TABLESPACE tbs1;
> CREATE TABLE originated (id int);
> INSERT INTO originated VALUES (1);
> CREATE UNIQUE INDEX ON originated(id) TABLESPACE tbs1;
> COMMIT;
..
> ERROR: could not open file "base/32833/32839": No such file or directory
> z3=# \dt+
> List of relations
> Schema | Name | Type | Owner | Persistence | Size | Description
> --------+------------+-------+----------+-------------+---------+-------------
> public | moved | table | postgres | permanent | 0 bytes |
> public | originated | table | postgres | permanent | 0 bytes |
>
> This happens even without placing on tablespace at all {for originated table , but no for moved on}, some major mishap is there (commit should guarantee correctness) or I'm tired and having sloppy fingers.
>
> 2) minor one testcase, still something is odd.
>
> drop tablespace tbs1;
> create tablespace tbs1 location '/tbs1';
> CREATE UNLOGGED TABLE t4 (a int) tablespace tbs1;
> CREATE UNLOGGED TABLE t5 (a int) tablespace tbs1;
> CREATE UNLOGGED TABLE t6 (a int) tablespace tbs1;
> CREATE TABLE t7 (a int) tablespace tbs1;
> insert into t7 values (1);
> insert into t5 values (1);
> insert into t6 values (1);
> \dt+
> List of relations
> Schema | Name | Type | Owner | Persistence | Size | Description
> --------+------+-------+----------+-------------+------------+-------------
> public | t4 | table | postgres | unlogged | 0 bytes |
> public | t5 | table | postgres | unlogged | 8192 bytes |
> public | t6 | table | postgres | unlogged | 8192 bytes |
> public | t7 | table | postgres | permanent | 8192 bytes |
> (4 rows)
>
> ALTER TABLE ALL IN TABLESPACE tbs1 set logged;
> ==> STILL WARNING: unrecognized node type: 349
> \dt+
> List of relations
> Schema | Name | Type | Owner | Persistence | Size | Description
> --------+------+-------+----------+-------------+------------+-------------
> public | t4 | table | postgres | permanent | 0 bytes |
> public | t5 | table | postgres | permanent | 8192 bytes |
> public | t6 | table | postgres | permanent | 8192 bytes |
> public | t7 | table | postgres | permanent | 8192 bytes |
>
> So it did rewrite however this warning seems to be unfixed. I've tested on e2c52beecdea152ca680a22ef35c6a7da55aa30f.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-21 11:04 ` Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-21 11:04 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Tue, 21 Dec 2021 17:13:21 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> Ugh! I completely forgot about TAP tests.. Thanks for the testing and
> sorry for the bugs.
>
> This is a bit big change so I need a bit of time before posting the
> next version.
I took a bit too long detour but the patch gets to pass make-world for
me.
In this version:
- When relation persistence is changed from logged to unlogged, buffer
persistence is flipped then an init-fork is created along with a mark
file for the fork (RelationCreateInitFork). The mark file is removed
at commit but left alone after a crash before commit. At the next
startup, ResetUnloggedRelationsInDbspaceDir() removes the init fork
file if it finds the mark file corresponding to the file.
- When relation persistence is changed from unlogged to logged, buffer
persistence is flipped then the exisging init-fork is marked to be
dropped at commit (RelationDropInitFork). Finally the whole content
is WAL-logged in the page-wise manner (RelationChangePersistence),
- The two operations above are repeatable within a transaction and
commit makes the last operation persist and rollback make the all
operations abandoned.
- Storage files are created along with a "mark" file for the
relfilenode. It behaves the same way to the above except the mark
files corresponds to the whole relfilenode.
- The at-commit operations this patch adds require to be WAL-logged so
they don't fit pendingDeletes list, which is executed after commit. I
added a new pending-work list pendingCleanups that is executed just
after pendingSyncs. (new in this version)
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-21 13:07 ` Jakub Wartak <[email protected]>
2021-12-22 06:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jakub Wartak @ 2021-12-21 13:07 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi Kyotaro,
> I took a bit too long detour but the patch gets to pass make-world for me.
Good news, v10 passes all the tests for me (including TAP recover ones). There's major problem I think:
drop table t6;
create unlogged table t6 (id bigint, t text);
create sequence s1;
insert into t6 select nextval('s1'), repeat('A', 1000) from generate_series(1, 100);
alter table t6 set logged;
select pg_sleep(1);
<--optional checkpoint, more on this later.
/usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile -m immediate stop
/usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile start
select count(*) from t6; -- shows 0 rows
But If I perform checkpoint before crash, data is there.. apparently the missing steps done by checkpointer
seem to help. If checkpoint is not done, then some peeking reveals that upon startup there is truncation (?!):
$ /usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile -m immediate stop
$ find /var/lib/pgsql/15/data/ -name '73741*' -ls
112723206 120 -rw------- 1 postgres postgres 122880 Dec 21 12:42 /var/lib/pgsql/15/data/base/73740/73741
112723202 24 -rw------- 1 postgres postgres 24576 Dec 21 12:42 /var/lib/pgsql/15/data/base/73740/73741_fsm
$ /usr/pgsql-15/bin/pg_ctl -D /var/lib/pgsql/15/data -l logfile start
waiting for server to start.... done
server started
$ find /var/lib/pgsql/15/data/ -name '73741*' -ls
112723206 0 -rw------- 1 postgres postgres 0 Dec 21 12:42 /var/lib/pgsql/15/data/base/73740/73741
112723202 16 -rw------- 1 postgres postgres 16384 Dec 21 12:42 /var/lib/pgsql/15/data/base/73740/73741_fsm
So what's suspicious is that 122880 -> 0 file size truncation. I've investigated WAL and it seems to contain TRUNCATE records
after logged FPI images, so when the crash recovery would kick in it probably clears this table (while it shouldn't).
However if I perform CHECKPOINT just before crash the WAL stream contains just RUNNING_XACTS and CHECKPOINT_ONLINE
redo records, this probably prevents truncating. I'm newbie here so please take this theory with grain of salt, it can be
something completely different.
-J.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-22 06:13 ` Kyotaro Horiguchi <[email protected]>
2021-12-22 08:42 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-22 06:13 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Hello, Jakub.
At Tue, 21 Dec 2021 13:07:28 +0000, Jakub Wartak <[email protected]> wrote in
> So what's suspicious is that 122880 -> 0 file size truncation. I've investigated WAL and it seems to contain TRUNCATE records
> after logged FPI images, so when the crash recovery would kick in it probably clears this table (while it shouldn't).
Darn.. It is too silly that I wrongly issued truncate records for the
target relation of the function (rel) instaed of the relation on which
we're currently operating at that time (r).
> However if I perform CHECKPOINT just before crash the WAL stream contains just RUNNING_XACTS and CHECKPOINT_ONLINE
> redo records, this probably prevents truncating. I'm newbie here so please take this theory with grain of salt, it can be
> something completely different.
It is because the WAL records are inconsistent with the on-disk state.
After a crash before a checkpoint after the SET LOGGED, recovery ends with
recoverying the broken WAL records, but after that the on-disk state
is persisted and the broken WAL records are not replayed.
The following fix works.
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -5478,7 +5478,7 @@ RelationChangePersistence(AlteredTableInfo *tab, char persistence,
xl_smgr_truncate xlrec;
xlrec.blkno = 0;
- xlrec.rnode = rel->rd_node;
+ xlrec.rnode = r->rd_node;
xlrec.flags = SMGR_TRUNCATE_ALL;
I made another change in this version. Previously only btree among all
index AMs was processed in the in-place manner. In this version we do
that all AMs except GiST. Maybe if gistGetFakeLSN behaved the same
way for permanent and unlogged indexes, we could skip index rebuild in
exchange of some extra WAL records emitted while it is unlogged.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* RE: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-22 06:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-22 08:42 ` Jakub Wartak <[email protected]>
2021-12-23 06:01 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jakub Wartak @ 2021-12-22 08:42 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi Kyotaro,
> At Tue, 21 Dec 2021 13:07:28 +0000, Jakub Wartak
> <[email protected]> wrote in
> > So what's suspicious is that 122880 -> 0 file size truncation. I've
> > investigated WAL and it seems to contain TRUNCATE records after logged
> FPI images, so when the crash recovery would kick in it probably clears this
> table (while it shouldn't).
>
> Darn.. It is too silly that I wrongly issued truncate records for the target
> relation of the function (rel) instaed of the relation on which we're currently
> operating at that time (r).
>
> [..]
> The following fix works.
Cool, I have verified basic stuff that was coming to my mind, now even cfbot is happy with v11, You should happy too I hope :)
> I made another change in this version. Previously only btree among all index
> AMs was processed in the in-place manner. In this version we do that all
> AMs except GiST. Maybe if gistGetFakeLSN behaved the same way for
> permanent and unlogged indexes, we could skip index rebuild in exchange of
> some extra WAL records emitted while it is unlogged.
I think there's slight omission:
-- unlogged table -> logged with GiST:
DROP TABLE IF EXISTS testcase;
CREATE UNLOGGED TABLE testcase(geom geometry not null);
CREATE INDEX idx_testcase_gist ON testcase USING gist(geom);
INSERT INTO testcase(geom) SELECT ST_Buffer(ST_SetSRID(ST_MakePoint(-1.0, 2.0),4326), 0.0001);
ALTER TABLE testcase SET LOGGED;
-- crashes with:
(gdb) where
#0 reindex_index (indexId=indexId@entry=65541, skip_constraint_checks=skip_constraint_checks@entry=true, persistence=persistence@entry=112 'p', params=params@entry=0x0) at index.c:3521
#1 0x000000000062f494 in RelationChangePersistence (tab=tab@entry=0x1947258, persistence=112 'p', lockmode=lockmode@entry=8) at tablecmds.c:5434
#2 0x0000000000642819 in ATRewriteTables (context=0x7ffc19c04520, lockmode=<optimized out>, wqueue=0x7ffc19c04388, parsetree=0x1925ec8) at tablecmds.c:5644
[..]
#10 0x00000000007f078f in exec_simple_query (query_string=0x1925340 "ALTER TABLE testcase SET LOGGED;") at postgres.c:1215
apparently reindex_index() params cannot be NULL - the same happens with switching persistent
table to unlogged one too (with GiST).
I'll also try to give another shot to the patch early next year - as we are starting long Christmas/holiday break here
- with verifying WAL for GiST and more advanced setup (more crashes, and standby/archiving/barman to see
how it's possible to use wal_level=minimal <-> replica transitions).
-J.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-22 06:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-22 08:42 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
@ 2021-12-23 06:01 ` Kyotaro Horiguchi <[email protected]>
2021-12-23 06:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-23 06:01 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Wed, 22 Dec 2021 08:42:14 +0000, Jakub Wartak <[email protected]> wrote in
> I think there's slight omission:
...
> apparently reindex_index() params cannot be NULL - the same happens with switching persistent
Hmm. a3dc926009 has changed the interface. (But the name is also
changed after that.)
-reindex_relation(Oid relid, int flags, int options)
+reindex_relation(Oid relid, int flags, ReindexParams *params)
> I'll also try to give another shot to the patch early next year - as we are starting long Christmas/holiday break here
> - with verifying WAL for GiST and more advanced setup (more crashes, and standby/archiving/barman to see
> how it's possible to use wal_level=minimal <-> replica transitions).
Thanks. I added TAP test to excecise the in-place persistence change.
have a nice holiday, Jakub!
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-22 06:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-22 08:42 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-23 06:01 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2021-12-23 06:33 ` Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2021-12-23 06:33 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Thu, 23 Dec 2021 15:01:41 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> I added TAP test to excecise the in-place persistence change.
We don't need a base table for every index. TAP test revised.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
@ 2020-11-13 07:47 ` Kyotaro Horiguchi <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-13 07:47 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 13 Nov 2020 06:43:13 +0000, "[email protected]" <[email protected]> wrote in
> Hi Horiguchi-san,
>
>
> Thank you for making a patch so quickly. I've started looking at it.
>
> What makes you think this is a PoC? Documentation and test cases? If there's something you think that doesn't work or are concerned about, can you share it?
The latest version is heavily revised and is given much comment so it
might have exited from PoC state. The necessity of documentation is
doubtful since this patch doesn't user-facing behavior other than
speed. Some tests are required especialy about recovery and
replication perspective but I haven't been able to make it. (One of
the tests needs to cause crash while a transaction is running.)
> Do you know the reason why data copy was done before? And, it may be odd for me to ask this, but I think I saw someone referred to the past discussion that eliminating data copy is difficult due to some processing at commit. I can't find it.
To imagine that, just because it is simpler considering rollback and
code sharing, and maybe no one have been complained that SET
LOGGED/UNLOGGED looks taking a long time than required/expected.
The current implement is simple. It's enough to just discard old or
new relfilenode according to the current transaction ends with commit
or abort. Tweaking of relfilenode under use leads-in some skews in
some places. I used pendingDelete mechanism a bit complexified way
and a violated an abstraction (I think, calling AM-routines from
storage.c is not good.) and even introduce a new fork kind only to
mark a init fork as "not committed yet". There might be better way,
but I haven't find it.
(The patch scans all shared buffer blocks for each relation).
> (1)
> @@ -168,6 +168,8 @@ extern PGDLLIMPORT int32 *LocalRefCount;
> */
> #define BufferGetPage(buffer) ((Page)BufferGetBlock(buffer))
>
> +struct SmgrRelationData;
>
> This declaration is already in the file:
>
> /* forward declared, to avoid having to expose buf_internals.h here */
> struct WritebackContext;
>
> /* forward declared, to avoid including smgr.h here */
> struct SMgrRelationData;
Hmmm. Nice chatch. And will fix in the next version.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* [PATCH v13 05/20] meson: prereq: Add src/tools/gen_export.pl
@ 2022-01-20 07:36 Andres Freund <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andres Freund @ 2022-01-20 07:36 UTC (permalink / raw)
Currently the logic is all in src/Makefile.shlib. This adds a sketch of a
generation script that can be used from meson.
Author: Andres Freund <[email protected]>
Reviewed-By: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/tools/gen_export.pl | 81 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 81 insertions(+)
create mode 100644 src/tools/gen_export.pl
diff --git a/src/tools/gen_export.pl b/src/tools/gen_export.pl
new file mode 100644
index 00000000000..68b3ab86614
--- /dev/null
+++ b/src/tools/gen_export.pl
@@ -0,0 +1,81 @@
+use strict;
+use warnings;
+use Getopt::Long;
+
+my $format;
+my $libname;
+my $input;
+my $output;
+
+GetOptions(
+ 'format:s' => \$format,
+ 'libname:s' => \$libname,
+ 'input:s' => \$input,
+ 'output:s' => \$output) or die "wrong arguments";
+
+if (not ($format eq 'aix' or $format eq 'darwin' or $format eq 'gnu' or $format eq 'win'))
+{
+ die "$0: $format is not yet handled (only aix, darwin, gnu, win are)\n";
+}
+
+open(my $input_handle, '<', $input)
+ or die "$0: could not open input file '$input': $!\n";
+
+open(my $output_handle, '>', $output)
+ or die "$0: could not open output file '$output': $!\n";
+
+
+if ($format eq 'gnu')
+{
+ print $output_handle "{
+ global:
+";
+}
+elsif ($format eq 'win')
+{
+ # XXX: Looks like specifying LIBRARY $libname is optional, which makes it
+ # easier to build a generic command for generating export files...
+ if ($libname)
+ {
+ print $output_handle "LIBRARY $libname\n";
+ }
+ print $output_handle "EXPORTS\n";
+}
+
+while (<$input_handle>)
+{
+ if (/^#/)
+ {
+ # don't do anything with a comment
+ }
+ elsif (/^(\S+)\s+(\S+)/)
+ {
+ if ($format eq 'aix')
+ {
+ print $output_handle "$1\n";
+ }
+ elsif ($format eq 'darwin')
+ {
+ print $output_handle "_$1\n";
+ }
+ elsif ($format eq 'gnu')
+ {
+ print $output_handle " $1;\n";
+ }
+ elsif ($format eq 'win')
+ {
+ print $output_handle "$1 @ $2\n";
+ }
+ }
+ else
+ {
+ die "$0: unexpected line $_\n";
+ }
+}
+
+if ($format eq 'gnu')
+{
+ print $output_handle " local: *;
+};
+";
+}
--
2.37.3.542.gdd3f6c4cae
--4uoxke3bx6ymzqvl
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0006-meson-prereq-Refactor-PG_TEST_EXTRA-logic-in-aut.patch"
^ permalink raw reply [nested|flat] 103+ messages in thread
* [PATCH v11 4/9] meson: prereq: Add src/tools/gen_export.pl
@ 2022-01-20 07:36 Andres Freund <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andres Freund @ 2022-01-20 07:36 UTC (permalink / raw)
Currently the logic is all in src/Makefile.shlib. This adds a sketch of a
generation script that can be used from meson.
---
src/tools/gen_export.pl | 83 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 83 insertions(+)
create mode 100644 src/tools/gen_export.pl
diff --git a/src/tools/gen_export.pl b/src/tools/gen_export.pl
new file mode 100644
index 00000000000..6a8b196ad89
--- /dev/null
+++ b/src/tools/gen_export.pl
@@ -0,0 +1,83 @@
+use strict;
+use warnings;
+use Getopt::Long;
+
+my $format;
+my $libname;
+my $input;
+my $output;
+
+GetOptions(
+ 'format:s' => \$format,
+ 'libname:s' => \$libname,
+ 'input:s' => \$input,
+ 'output:s' => \$output) or die "wrong arguments";
+
+if (not ($format eq 'aix' or $format eq 'darwin' or $format eq 'gnu' or $format eq 'win'))
+{
+ die "$0: $format is not yet handled (only aix, darwin, gnu, win are)\n";
+}
+
+open(my $input_handle, '<', $input)
+ or die "$0: could not open input file '$input': $!\n";
+
+open(my $output_handle, '>', $output)
+ or die "$0: could not open output file '$output': $!\n";
+
+
+if ($format eq 'gnu')
+{
+ print $output_handle "{
+ global:
+";
+}
+elsif ($format eq 'win')
+{
+ # XXX: Looks like specifying LIBRARY $libname is optional, which makes it
+ # easier to build a generic command for generating export files...
+ if ($libname)
+ {
+ print $output_handle "LIBRARY $libname\n";
+ }
+ print $output_handle "EXPORTS\n";
+}
+
+while (<$input_handle>)
+{
+ if (/^#/)
+ {
+ # don't do anything with a comment
+ }
+ elsif (/^([^\s]+)\s+([^\s]+)/)
+ {
+ if ($format eq 'aix')
+ {
+ print $output_handle " $1\n";
+ }
+ elsif ($format eq 'darwin')
+ {
+ print $output_handle " _$1\n";
+ }
+ elsif ($format eq 'gnu')
+ {
+ print $output_handle " $1;\n";
+ }
+ elsif ($format eq 'win')
+ {
+ print $output_handle " $1 @ $2\n";
+ }
+ }
+ else
+ {
+ die "$0: unexpected line $_\n";
+ }
+}
+
+if ($format eq 'gnu')
+{
+ print $output_handle " local: *;
+};
+";
+}
+
+exit(0);
--
2.37.0.3.g30cc8d0f14
--epod4qqm5dwbci5r
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v11-0005-meson-prereq-Refactor-PG_TEST_EXTRA-logic-in-aut.patch"
^ permalink raw reply [nested|flat] 103+ messages in thread
* [PATCH v13 05/20] meson: prereq: Add src/tools/gen_export.pl
@ 2022-01-20 07:36 Andres Freund <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andres Freund @ 2022-01-20 07:36 UTC (permalink / raw)
Currently the logic is all in src/Makefile.shlib. This adds a sketch of a
generation script that can be used from meson.
Author: Andres Freund <[email protected]>
Reviewed-By: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/tools/gen_export.pl | 81 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 81 insertions(+)
create mode 100644 src/tools/gen_export.pl
diff --git a/src/tools/gen_export.pl b/src/tools/gen_export.pl
new file mode 100644
index 00000000000..68b3ab86614
--- /dev/null
+++ b/src/tools/gen_export.pl
@@ -0,0 +1,81 @@
+use strict;
+use warnings;
+use Getopt::Long;
+
+my $format;
+my $libname;
+my $input;
+my $output;
+
+GetOptions(
+ 'format:s' => \$format,
+ 'libname:s' => \$libname,
+ 'input:s' => \$input,
+ 'output:s' => \$output) or die "wrong arguments";
+
+if (not ($format eq 'aix' or $format eq 'darwin' or $format eq 'gnu' or $format eq 'win'))
+{
+ die "$0: $format is not yet handled (only aix, darwin, gnu, win are)\n";
+}
+
+open(my $input_handle, '<', $input)
+ or die "$0: could not open input file '$input': $!\n";
+
+open(my $output_handle, '>', $output)
+ or die "$0: could not open output file '$output': $!\n";
+
+
+if ($format eq 'gnu')
+{
+ print $output_handle "{
+ global:
+";
+}
+elsif ($format eq 'win')
+{
+ # XXX: Looks like specifying LIBRARY $libname is optional, which makes it
+ # easier to build a generic command for generating export files...
+ if ($libname)
+ {
+ print $output_handle "LIBRARY $libname\n";
+ }
+ print $output_handle "EXPORTS\n";
+}
+
+while (<$input_handle>)
+{
+ if (/^#/)
+ {
+ # don't do anything with a comment
+ }
+ elsif (/^(\S+)\s+(\S+)/)
+ {
+ if ($format eq 'aix')
+ {
+ print $output_handle "$1\n";
+ }
+ elsif ($format eq 'darwin')
+ {
+ print $output_handle "_$1\n";
+ }
+ elsif ($format eq 'gnu')
+ {
+ print $output_handle " $1;\n";
+ }
+ elsif ($format eq 'win')
+ {
+ print $output_handle "$1 @ $2\n";
+ }
+ }
+ else
+ {
+ die "$0: unexpected line $_\n";
+ }
+}
+
+if ($format eq 'gnu')
+{
+ print $output_handle " local: *;
+};
+";
+}
--
2.37.3.542.gdd3f6c4cae
--4uoxke3bx6ymzqvl
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0006-meson-prereq-Refactor-PG_TEST_EXTRA-logic-in-aut.patch"
^ permalink raw reply [nested|flat] 103+ messages in thread
* [PATCH v12 04/15] meson: prereq: Add src/tools/gen_export.pl
@ 2022-01-20 07:36 Andres Freund <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andres Freund @ 2022-01-20 07:36 UTC (permalink / raw)
Currently the logic is all in src/Makefile.shlib. This adds a sketch of a
generation script that can be used from meson.
Author: Andres Freund <[email protected]>
Reviewed-By: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/tools/gen_export.pl | 81 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 81 insertions(+)
create mode 100644 src/tools/gen_export.pl
diff --git a/src/tools/gen_export.pl b/src/tools/gen_export.pl
new file mode 100644
index 00000000000..3a368c85381
--- /dev/null
+++ b/src/tools/gen_export.pl
@@ -0,0 +1,81 @@
+use strict;
+use warnings;
+use Getopt::Long;
+
+my $format;
+my $libname;
+my $input;
+my $output;
+
+GetOptions(
+ 'format:s' => \$format,
+ 'libname:s' => \$libname,
+ 'input:s' => \$input,
+ 'output:s' => \$output) or die "wrong arguments";
+
+if (not ($format eq 'aix' or $format eq 'darwin' or $format eq 'gnu' or $format eq 'win'))
+{
+ die "$0: $format is not yet handled (only aix, darwin, gnu, win are)\n";
+}
+
+open(my $input_handle, '<', $input)
+ or die "$0: could not open input file '$input': $!\n";
+
+open(my $output_handle, '>', $output)
+ or die "$0: could not open output file '$output': $!\n";
+
+
+if ($format eq 'gnu')
+{
+ print $output_handle "{
+ global:
+";
+}
+elsif ($format eq 'win')
+{
+ # XXX: Looks like specifying LIBRARY $libname is optional, which makes it
+ # easier to build a generic command for generating export files...
+ if ($libname)
+ {
+ print $output_handle "LIBRARY $libname\n";
+ }
+ print $output_handle "EXPORTS\n";
+}
+
+while (<$input_handle>)
+{
+ if (/^#/)
+ {
+ # don't do anything with a comment
+ }
+ elsif (/^(\S+)\s+(\S+)/)
+ {
+ if ($format eq 'aix')
+ {
+ print $output_handle " $1\n";
+ }
+ elsif ($format eq 'darwin')
+ {
+ print $output_handle " _$1\n";
+ }
+ elsif ($format eq 'gnu')
+ {
+ print $output_handle " $1;\n";
+ }
+ elsif ($format eq 'win')
+ {
+ print $output_handle " $1 @ $2\n";
+ }
+ }
+ else
+ {
+ die "$0: unexpected line $_\n";
+ }
+}
+
+if ($format eq 'gnu')
+{
+ print $output_handle " local: *;
+};
+";
+}
--
2.37.0.3.g30cc8d0f14
--3jz64y4qgcz5d4ku
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v12-0005-meson-prereq-Refactor-PG_TEST_EXTRA-logic-in-aut.patch"
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
@ 2022-02-16 21:07 Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-02-16 21:07 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Wed, Feb 16, 2022 at 12:46 PM Jeevan Ladhe <[email protected]> wrote:
> So, I went ahead and have now also implemented client side decompression
> for zstd.
>
> Robert separated[1] the ZSTD configure switch from my original patch
> of server side compression and also added documentation related to
> the switch. I have included that patch here in the patch series for
> simplicity.
>
> The server side compression patch
> 0002-ZSTD-add-server-side-compression-support.patch has also taken care
> of Justin Pryzby's comments[2]. Also, made changes to pg_basebackup help
> as suggested by Álvaro Herrera.
The first hunk of the documentation changes is missing a comma between
gzip and lz4.
+ * At the start of each archive we reset the state to start a new
+ * compression operation. The parameters are sticky and they would stick
+ * around as we are resetting with option ZSTD_reset_session_only.
I don't think "would" is what you mean here. If you say something
would stick around, that means it could be that way it isn't. ("I
would go to the store and buy some apples, but I know they don't have
any so there's no point.") I think you mean "will".
- printf(_(" -Z,
--compress={[{client,server}-]gzip,lz4,none}[:LEVEL] or [LEVEL]\n"
- " compress tar output with given
compression method or level\n"));
+ printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n"));
+ printf(_(" -Z, --compress=none\n"));
You deleted a line that you should have preserved here.
Overall there doesn't seem to be much to complain about here on a
first read-through. It will be good if we can also fix
CreateWalTarMethod to support LZ4 and ZSTD.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-02-17 01:46 ` Jeevan Ladhe <[email protected]>
2022-03-04 08:31 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 2 replies; 103+ messages in thread
From: Jeevan Ladhe @ 2022-02-17 01:46 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Thanks for the comments Robert. I have addressed your comments in the
attached patch v13-0002-ZSTD-add-server-side-compression-support.patch.
Rest of the patches are similar to v12, but just bumped the version number.
> It will be good if we can also fix
> CreateWalTarMethod to support LZ4 and ZSTD.
Ok we will see, either Dipesh or I will take care of it.
Regards,
Jeevan Ladhe
On Thu, 17 Feb 2022 at 02:37, Robert Haas <[email protected]> wrote:
> On Wed, Feb 16, 2022 at 12:46 PM Jeevan Ladhe <[email protected]>
> wrote:
> > So, I went ahead and have now also implemented client side decompression
> > for zstd.
> >
> > Robert separated[1] the ZSTD configure switch from my original patch
> > of server side compression and also added documentation related to
> > the switch. I have included that patch here in the patch series for
> > simplicity.
> >
> > The server side compression patch
> > 0002-ZSTD-add-server-side-compression-support.patch has also taken care
> > of Justin Pryzby's comments[2]. Also, made changes to pg_basebackup help
> > as suggested by Álvaro Herrera.
>
> The first hunk of the documentation changes is missing a comma between
> gzip and lz4.
>
> + * At the start of each archive we reset the state to start a new
> + * compression operation. The parameters are sticky and they would
> stick
> + * around as we are resetting with option ZSTD_reset_session_only.
>
> I don't think "would" is what you mean here. If you say something
> would stick around, that means it could be that way it isn't. ("I
> would go to the store and buy some apples, but I know they don't have
> any so there's no point.") I think you mean "will".
>
> - printf(_(" -Z,
> --compress={[{client,server}-]gzip,lz4,none}[:LEVEL] or [LEVEL]\n"
> - " compress tar output with given
> compression method or level\n"));
> + printf(_(" -Z,
> --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n"));
> + printf(_(" -Z, --compress=none\n"));
>
> You deleted a line that you should have preserved here.
>
> Overall there doesn't seem to be much to complain about here on a
> first read-through. It will be good if we can also fix
> CreateWalTarMethod to support LZ4 and ZSTD.
>
> --
> Robert Haas
> EDB: http://www.enterprisedb.com
>
Attachments:
[application/octet-stream] v13-0004-ZSTD-add-client-side-decompression-support.patch (7.1K, ../../CANm22Cim1JFzvbk+N7EynYRpovmhhAU5btxC=QUDDtzccnKjDQ@mail.gmail.com/3-v13-0004-ZSTD-add-client-side-decompression-support.patch)
download | inline diff:
From fb7c1e49afaea669f2baa7f05ed2eaf5ae003d81 Mon Sep 17 00:00:00 2001
From: Jeevan Ladhe <[email protected]>
Date: Wed, 16 Feb 2022 22:51:47 +0530
Subject: [PATCH 4/4] ZSTD: add client-side decompression support.
ZSTD decompression of a backup compressed on the server can be
performed on the client using pg_basebackup -Fp --compress server-lz4.
Example:
pg_basebackup -D /tmp/zstd_C_D -Fp -Xfetch --compress=server-zstd:7
---
src/bin/pg_basebackup/bbstreamer.h | 1 +
src/bin/pg_basebackup/bbstreamer_zstd.c | 133 +++++++++++++++++++++++
src/bin/pg_basebackup/pg_basebackup.c | 2 +
src/bin/pg_verifybackup/t/009_extract.pl | 5 +
4 files changed, 141 insertions(+)
mode change 100644 => 100755 src/bin/pg_verifybackup/t/009_extract.pl
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
index bfc624a863..02d4c05df6 100644
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ b/src/bin/pg_basebackup/bbstreamer.h
@@ -211,6 +211,7 @@ extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
int compresslevel);
+extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index 0b20267cf4..83b59d63ba 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -27,6 +27,7 @@ typedef struct bbstreamer_zstd_frame
bbstreamer base;
ZSTD_CCtx *cctx;
+ ZSTD_DCtx *dctx;
ZSTD_outBuffer zstd_outBuf;
} bbstreamer_zstd_frame;
@@ -42,6 +43,19 @@ const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
.finalize = bbstreamer_zstd_compressor_finalize,
.free = bbstreamer_zstd_compressor_free
};
+
+static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context);
+static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
+static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+
+const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
+ .content = bbstreamer_zstd_decompressor_content,
+ .finalize = bbstreamer_zstd_decompressor_finalize,
+ .free = bbstreamer_zstd_decompressor_free
+};
#endif
/*
@@ -200,3 +214,122 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
pfree(streamer);
}
#endif
+
+/*
+ * Create a new base backup streamer that performs decompression of zstd
+ * compressed blocks.
+ */
+bbstreamer *
+bbstreamer_zstd_decompressor_new(bbstreamer *next)
+{
+#ifdef HAVE_LIBZSTD
+ bbstreamer_zstd_frame *streamer;
+
+ Assert(next != NULL);
+
+ streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+ *((const bbstreamer_ops **) &streamer->base.bbs_ops) =
+ &bbstreamer_zstd_decompressor_ops;
+
+ streamer->base.bbs_next = next;
+ initStringInfo(&streamer->base.bbs_buffer);
+ enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize());
+
+ streamer->dctx = ZSTD_createDCtx();
+ if (!streamer->dctx)
+ {
+ pg_log_error("could not create zstd decompression context");
+ exit(1);
+ }
+
+ /* Initialize the ZSTD output buffer. */
+ streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
+ streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
+ streamer->zstd_outBuf.pos = 0;
+
+ return &streamer->base;
+#else
+ pg_log_error("this build does not support compression");
+ exit(1);
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+/*
+ * Decompress the input data to output buffer until we run out of input
+ * data. Each time the output buffer is full, pass on the decompressed data
+ * to the next streamer.
+ */
+static void
+bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ ZSTD_inBuffer inBuf = {data, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t ret;
+
+ /*
+ * If output buffer is full then forward the content to next streamer
+ * and update the output buffer.
+ */
+ if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, member,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ context);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ ret = ZSTD_decompressStream(mystreamer->dctx,
+ &mystreamer->zstd_outBuf, &inBuf);
+
+ if (ZSTD_isError(ret))
+ pg_log_error("could not decompress data: %s", ZSTD_getErrorName(ret));
+ }
+}
+
+/*
+ * End-of-stream processing.
+ */
+static void
+bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ /*
+ * End of the stream, if there is some pending data in output buffers then
+ * we must forward it to next streamer.
+ */
+ if (mystreamer->zstd_outBuf.pos > 0)
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->base.bbs_buffer.data,
+ mystreamer->base.bbs_buffer.maxlen,
+ BBSTREAMER_UNKNOWN);
+
+ bbstreamer_finalize(mystreamer->base.bbs_next);
+}
+
+/*
+ * Free memory.
+ */
+static void
+bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ bbstreamer_free(streamer->bbs_next);
+ ZSTD_freeDCtx(mystreamer->dctx);
+ pfree(streamer->bbs_buffer.data);
+ pfree(streamer);
+}
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 18bd0df9a5..cef66d3e9e 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -1333,6 +1333,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
streamer = bbstreamer_gzip_decompressor_new(streamer);
else if (compressmethod == COMPRESSION_LZ4)
streamer = bbstreamer_lz4_decompressor_new(streamer);
+ else if (compressmethod == COMPRESSION_ZSTD)
+ streamer = bbstreamer_zstd_decompressor_new(streamer);
}
/* Return the results. */
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
old mode 100644
new mode 100755
index c51cdf79f8..d30ba01742
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -31,6 +31,11 @@ my @test_configuration = (
'compression_method' => 'lz4',
'backup_flags' => ['--compress', 'server-lz4:5'],
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'server-zstd:5'],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
--
2.25.1
[application/octet-stream] v13-0001-Add-support-for-building-with-ZSTD.patch (16.3K, ../../CANm22Cim1JFzvbk+N7EynYRpovmhhAU5btxC=QUDDtzccnKjDQ@mail.gmail.com/4-v13-0001-Add-support-for-building-with-ZSTD.patch)
download | inline diff:
From cfa0448be55b7b2f9131ffec656a69f1779e0f5e Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Wed, 16 Feb 2022 10:36:36 -0500
Subject: [PATCH 1/4] Add support for building with ZSTD.
This commit doesn't actually add anything that uses ZSTD; that will be
done separately. It just puts the basic infrastructure into place.
Jeevan Ladhe and Robert Haas
---
configure | 271 ++++++++++++++++++++++++++++++
configure.ac | 33 ++++
doc/src/sgml/install-windows.sgml | 9 +
doc/src/sgml/installation.sgml | 9 +
src/Makefile.global.in | 1 +
src/include/pg_config.h.in | 9 +
src/tools/msvc/Solution.pm | 12 ++
src/tools/msvc/config_default.pl | 1 +
8 files changed, 345 insertions(+)
diff --git a/configure b/configure
index 9305555658..f07f689f1a 100755
--- a/configure
+++ b/configure
@@ -650,6 +650,7 @@ CFLAGS_ARMV8_CRC32C
CFLAGS_SSE42
have_win32_dbghelp
LIBOBJS
+ZSTD
LZ4
UUID_LIBS
LDAP_LIBS_BE
@@ -700,6 +701,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+ZSTD_LIBS
+ZSTD_CFLAGS
+with_zstd
LZ4_LIBS
LZ4_CFLAGS
with_lz4
@@ -869,6 +873,7 @@ with_libxslt
with_system_tzdata
with_zlib
with_lz4
+with_zstd
with_gnu_ld
with_ssl
with_openssl
@@ -898,6 +903,8 @@ XML2_CFLAGS
XML2_LIBS
LZ4_CFLAGS
LZ4_LIBS
+ZSTD_CFLAGS
+ZSTD_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1577,6 +1584,7 @@ Optional Packages:
use system time zone data in DIR
--without-zlib do not use Zlib
--with-lz4 build with LZ4 support
+ --with-zstd build with ZSTD support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1606,6 +1614,8 @@ Some influential environment variables:
XML2_LIBS linker flags for XML2, overriding pkg-config
LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
LZ4_LIBS linker flags for LZ4, overriding pkg-config
+ ZSTD_CFLAGS C compiler flags for ZSTD, overriding pkg-config
+ ZSTD_LIBS linker flags for ZSTD, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -9034,6 +9044,146 @@ fi
done
fi
+#
+# ZSTD
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with ZSTD support" >&5
+$as_echo_n "checking whether to build with ZSTD support... " >&6; }
+
+
+
+# Check whether --with-zstd was given.
+if test "${with_zstd+set}" = set; then :
+ withval=$with_zstd;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_ZSTD 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-zstd option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_zstd=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_zstd" >&5
+$as_echo "$with_zstd" >&6; }
+
+
+if test "$with_zstd" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for libzstd" >&5
+$as_echo_n "checking for libzstd... " >&6; }
+
+if test -n "$ZSTD_CFLAGS"; then
+ pkg_cv_ZSTD_CFLAGS="$ZSTD_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_ZSTD_CFLAGS=`$PKG_CONFIG --cflags "libzstd" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$ZSTD_LIBS"; then
+ pkg_cv_ZSTD_LIBS="$ZSTD_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_ZSTD_LIBS=`$PKG_CONFIG --libs "libzstd" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ ZSTD_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libzstd" 2>&1`
+ else
+ ZSTD_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libzstd" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$ZSTD_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (libzstd) were not met:
+
+$ZSTD_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables ZSTD_CFLAGS
+and ZSTD_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables ZSTD_CFLAGS
+and ZSTD_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/>.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ ZSTD_CFLAGS=$pkg_cv_ZSTD_CFLAGS
+ ZSTD_LIBS=$pkg_cv_ZSTD_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ # We only care about -I, -D, and -L switches;
+ # note that -lzstd will be added by AC_CHECK_LIB below.
+ for pgac_option in $ZSTD_CFLAGS; do
+ case $pgac_option in
+ -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";;
+ esac
+ done
+ for pgac_option in $ZSTD_LIBS; do
+ case $pgac_option in
+ -L*) LDFLAGS="$LDFLAGS $pgac_option";;
+ esac
+ done
+fi
#
# Assignments
#
@@ -13130,6 +13280,56 @@ fi
fi
+if test "$with_zstd" = yes ; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_compress in -lzstd" >&5
+$as_echo_n "checking for ZSTD_compress in -lzstd... " >&6; }
+if ${ac_cv_lib_zstd_ZSTD_compress+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lzstd $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_compress ();
+int
+main ()
+{
+return ZSTD_compress ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_zstd_ZSTD_compress=yes
+else
+ ac_cv_lib_zstd_ZSTD_compress=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_compress" >&5
+$as_echo "$ac_cv_lib_zstd_ZSTD_compress" >&6; }
+if test "x$ac_cv_lib_zstd_ZSTD_compress" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBZSTD 1
+_ACEOF
+
+ LIBS="-lzstd $LIBS"
+
+else
+ as_fn_error $? "library 'zstd' is required for ZSTD support" "$LINENO" 5
+fi
+
+fi
+
# Note: We can test for libldap_r only after we know PTHREAD_LIBS;
# also, on AIX, we may need to have openssl in LIBS for this step.
if test "$with_ldap" = yes ; then
@@ -13904,6 +14104,77 @@ done
fi
+if test -z "$ZSTD"; then
+ for ac_prog in zstd
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_ZSTD+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $ZSTD in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_ZSTD="$ZSTD" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_ZSTD="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+ZSTD=$ac_cv_path_ZSTD
+if test -n "$ZSTD"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ZSTD" >&5
+$as_echo "$ZSTD" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$ZSTD" && break
+done
+
+else
+ # Report the value of ZSTD in configure's output in all cases.
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD" >&5
+$as_echo_n "checking for ZSTD... " >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ZSTD" >&5
+$as_echo "$ZSTD" >&6; }
+fi
+
+if test "$with_zstd" = yes; then
+ for ac_header in zstd.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default"
+if test "x$ac_cv_header_zstd_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_ZSTD_H 1
+_ACEOF
+
+else
+ as_fn_error $? "zstd.h header file is required for ZSTD" "$LINENO" 5
+fi
+
+done
+
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 16167329fc..729b23fbea 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1056,6 +1056,30 @@ if test "$with_lz4" = yes; then
done
fi
+#
+# ZSTD
+#
+AC_MSG_CHECKING([whether to build with ZSTD support])
+PGAC_ARG_BOOL(with, zstd, no, [build with ZSTD support],
+ [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with ZSTD support. (--with-zstd)])])
+AC_MSG_RESULT([$with_zstd])
+AC_SUBST(with_zstd)
+
+if test "$with_zstd" = yes; then
+ PKG_CHECK_MODULES(ZSTD, libzstd)
+ # We only care about -I, -D, and -L switches;
+ # note that -lzstd will be added by AC_CHECK_LIB below.
+ for pgac_option in $ZSTD_CFLAGS; do
+ case $pgac_option in
+ -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";;
+ esac
+ done
+ for pgac_option in $ZSTD_LIBS; do
+ case $pgac_option in
+ -L*) LDFLAGS="$LDFLAGS $pgac_option";;
+ esac
+ done
+fi
#
# Assignments
#
@@ -1325,6 +1349,10 @@ if test "$with_lz4" = yes ; then
AC_CHECK_LIB(lz4, LZ4_compress_default, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])])
fi
+if test "$with_zstd" = yes ; then
+ AC_CHECK_LIB(zstd, ZSTD_compress, [], [AC_MSG_ERROR([library 'zstd' is required for ZSTD support])])
+fi
+
# Note: We can test for libldap_r only after we know PTHREAD_LIBS;
# also, on AIX, we may need to have openssl in LIBS for this step.
if test "$with_ldap" = yes ; then
@@ -1490,6 +1518,11 @@ if test "$with_lz4" = yes; then
AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])
fi
+PGAC_PATH_PROGS(ZSTD, zstd)
+if test "$with_zstd" = yes; then
+ AC_CHECK_HEADERS(zstd.h, [], [AC_MSG_ERROR([zstd.h header file is required for ZSTD])])
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index 30dd0c7f75..d2f63db3f2 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -307,6 +307,15 @@ $ENV{MSBFLAGS}="/m";
</para></listitem>
</varlistentry>
+ <varlistentry>
+ <term><productname>ZSTD</productname></term>
+ <listitem><para>
+ Required for supporting <productname>ZSTD</productname> compression
+ method. Binaries and source can be downloaded from
+ <ulink url="https://github.com/facebook/zstd/releases"></ulink>.
+ </para></listitem>
+ </varlistentry>
+
<varlistentry>
<term><productname>OpenSSL</productname></term>
<listitem><para>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 655095f3b1..c6190f6955 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -989,6 +989,15 @@ build-postgresql:
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-zstd</option></term>
+ <listitem>
+ <para>
+ Build with <productname>ZSTD</productname> compression support.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--with-ssl=<replaceable>LIBRARY</replaceable></option>
<indexterm>
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 9dcd54fcbd..c980444233 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -351,6 +351,7 @@ XGETTEXT = @XGETTEXT@
GZIP = gzip
BZIP2 = bzip2
LZ4 = @LZ4@
+ZSTD = @ZSTD@
DOWNLOAD = wget -O $@ --no-use-server-timestamps
#DOWNLOAD = curl -o $@
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 28a1f0e9f0..1912cf35de 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -352,6 +352,9 @@
/* Define to 1 if you have the `z' library (-lz). */
#undef HAVE_LIBZ
+/* Define to 1 if you have the `zstd' library (-lzstd). */
+#undef HAVE_LIBZSTD
+
/* Define to 1 if you have the `link' function. */
#undef HAVE_LINK
@@ -718,6 +721,9 @@
/* Define to 1 if the assembler supports X86_64's POPCNTQ instruction. */
#undef HAVE_X86_64_POPCNTQ
+/* Define to 1 if you have the <zstd.h> header file. */
+#undef HAVE_ZSTD_H
+
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
@@ -949,6 +955,9 @@
/* Define to select Win32-style shared memory. */
#undef USE_WIN32_SHARED_MEMORY
+/* Define to 1 to build with ZSTD support. (--with-zstd) */
+#undef USE_ZSTD
+
/* Define to 1 if `wcstombs_l' requires <xlocale.h>. */
#undef WCSTOMBS_L_IN_XLOCALE
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index e6f20679dc..087acfbaa1 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -539,6 +539,12 @@ sub GenerateFiles
$define{HAVE_LZ4_H} = 1;
$define{USE_LZ4} = 1;
}
+ if ($self->{options}->{zstd})
+ {
+ $define{HAVE_LIBZSTD} = 1;
+ $define{HAVE_ZSTD_H} = 1;
+ $define{USE_ZSTD} = 1;
+ }
if ($self->{options}->{openssl})
{
$define{USE_OPENSSL} = 1;
@@ -1081,6 +1087,11 @@ sub AddProject
$proj->AddIncludeDir($self->{options}->{lz4} . '\include');
$proj->AddLibrary($self->{options}->{lz4} . '\lib\liblz4.lib');
}
+ if ($self->{options}->{zstd})
+ {
+ $proj->AddIncludeDir($self->{options}->{zstd} . '\include');
+ $proj->AddLibrary($self->{options}->{zstd} . '\lib\libzstd.lib');
+ }
if ($self->{options}->{uuid})
{
$proj->AddIncludeDir($self->{options}->{uuid} . '\include');
@@ -1193,6 +1204,7 @@ sub GetFakeConfigure
$cfg .= ' --with-libxml' if ($self->{options}->{xml});
$cfg .= ' --with-libxslt' if ($self->{options}->{xslt});
$cfg .= ' --with-lz4' if ($self->{options}->{lz4});
+ $cfg .= ' --with-zstd' if ($self->{options}->{zstd});
$cfg .= ' --with-gssapi' if ($self->{options}->{gss});
$cfg .= ' --with-icu' if ($self->{options}->{icu});
$cfg .= ' --with-tcl' if ($self->{options}->{tcl});
diff --git a/src/tools/msvc/config_default.pl b/src/tools/msvc/config_default.pl
index 7a9b00be72..186849a09a 100644
--- a/src/tools/msvc/config_default.pl
+++ b/src/tools/msvc/config_default.pl
@@ -15,6 +15,7 @@ our $config = {
gss => undef, # --with-gssapi=<path>
icu => undef, # --with-icu=<path>
lz4 => undef, # --with-lz4=<path>
+ zstd => undef, # --with-zstd=<path>
nls => undef, # --enable-nls=<path>
tap_tests => undef, # --enable-tap-tests
tcl => undef, # --with-tcl=<path>
--
2.25.1
[application/octet-stream] v13-0002-ZSTD-add-server-side-compression-support.patch (20.5K, ../../CANm22Cim1JFzvbk+N7EynYRpovmhhAU5btxC=QUDDtzccnKjDQ@mail.gmail.com/5-v13-0002-ZSTD-add-server-side-compression-support.patch)
download | inline diff:
From 0f0770583989fc55afa8003046ea9b85af2142b3 Mon Sep 17 00:00:00 2001
From: Jeevan Ladhe <[email protected]>
Date: Thu, 17 Feb 2022 06:55:53 +0530
Subject: [PATCH 2/4] ZSTD: add server-side compression support.
This patch introduces --compress=server-zstd[:LEVEL]
Add tap test.
Add config option --with-zstd.
Add documentation for ZSTD option.
Add pg_basebackup help for ZSTD option.
Example:
pg_basebackup -t server:/tmp/data_test -Xnone --compress=server-zstd:4
---
doc/src/sgml/protocol.sgml | 7 +-
doc/src/sgml/ref/pg_basebackup.sgml | 38 +--
src/backend/replication/Makefile | 1 +
src/backend/replication/basebackup.c | 7 +-
src/backend/replication/basebackup_zstd.c | 294 ++++++++++++++++++++++
src/bin/pg_basebackup/pg_basebackup.c | 19 +-
src/bin/pg_basebackup/pg_receivewal.c | 4 +
src/bin/pg_basebackup/walmethods.h | 1 +
src/bin/pg_verifybackup/Makefile | 1 +
src/bin/pg_verifybackup/t/008_untar.pl | 9 +
src/include/replication/basebackup_sink.h | 1 +
11 files changed, 359 insertions(+), 23 deletions(-)
create mode 100644 src/backend/replication/basebackup_zstd.c
mode change 100644 => 100755 src/bin/pg_verifybackup/t/008_untar.pl
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 1c5ab00879..8fe638767d 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2724,8 +2724,8 @@ The commands accepted in replication mode are:
<listitem>
<para>
Instructs the server to compress the backup using the specified
- method. Currently, the supported methods are <literal>gzip</literal>
- and <literal>lz4</literal>.
+ method. Currently, the supported methods are <literal>gzip</literal>,
+ <literal>lz4</literal>, and <literal>zstd</literal>.
</para>
</listitem>
</varlistentry>
@@ -2737,7 +2737,8 @@ The commands accepted in replication mode are:
Specifies the compression level to be used. This should only be
used in conjunction with the <literal>COMPRESSION</literal> option.
For <literal>gzip</literal> the value should be an integer between 1
- and 9, and for <literal>lz4</literal> it should be between 1 and 12.
+ and 9, for <literal>lz4</literal> between 1 and 12, and for
+ <literal>zstd</literal> it should be between 1 and 22.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 53aa40dcd1..4cf28a2a61 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -417,30 +417,32 @@ PostgreSQL documentation
specify <literal>-Xfetch</literal>.
</para>
<para>
- The compression method can be set to <literal>gzip</literal> or
- <literal>lz4</literal>, or <literal>none</literal> for no
- compression. A compression level can be optionally specified, by
- appending the level number after a colon (<literal>:</literal>). If no
- level is specified, the default compression level will be used. If
- only a level is specified without mentioning an algorithm,
- <literal>gzip</literal> compression will be used if the level is
- greater than 0, and no compression will be used if the level is 0.
- </para>
- <para>
- When the tar format is used with <literal>gzip</literal> or
- <literal>lz4</literal>, the suffix <filename>.gz</filename> or
- <filename>.lz4</filename> will automatically be added to all tar
- filenames. When the plain format is used, client-side compression may
- not be specified, but it is still possible to request server-side
- compression. If this is done, the server will compress the backup for
- transmission, and the client will decompress and extract it.
+ The compression method can be set to <literal>gzip</literal>,
+ <literal>lz4</literal>, <literal>zstd</literal>, or
+ <literal>none</literal> for no compression. A compression level can
+ optionally be specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default
+ compression level will be used. If only a level is specified without
+ mentioning an algorithm, <literal>gzip</literal> compression will be
+ used if the level is greater than 0, and no compression will be used if
+ the level is 0.
+ </para>
+ <para>
+ When the tar format is used with <literal>gzip</literal>,
+ <literal>lz4</literal>, or <literal>zstd</literal>, the suffix
+ <filename>.gz</filename>, <filename>.lz4</filename>, or
+ <filename>.zst</filename> respectively will be automatically added to
+ all tar filenames. When the plain format is used, client-side
+ compression may not be specified, but it is still possible to request
+ server-side compression. If this is done, the server will compress the
+ backup for transmission, and the client will decompress and extract it.
</para>
<para>
When this option is used in combination with
<literal>-Xstream</literal>, <literal>pg_wal.tar</literal> will
be compressed using <literal>gzip</literal> if client-side gzip
compression is selected, but will not be compressed if server-side
- compresion or LZ4 compresion is selected.
+ compression, LZ4, or ZSTD compression is selected.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile
index 74043ff331..2e6de7007f 100644
--- a/src/backend/replication/Makefile
+++ b/src/backend/replication/Makefile
@@ -20,6 +20,7 @@ OBJS = \
basebackup_copy.o \
basebackup_gzip.o \
basebackup_lz4.o \
+ basebackup_zstd.o \
basebackup_progress.o \
basebackup_server.o \
basebackup_sink.o \
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 0bf28b55d7..2378ce5c5e 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -64,7 +64,8 @@ typedef enum
{
BACKUP_COMPRESSION_NONE,
BACKUP_COMPRESSION_GZIP,
- BACKUP_COMPRESSION_LZ4
+ BACKUP_COMPRESSION_LZ4,
+ BACKUP_COMPRESSION_ZSTD
} basebackup_compression_type;
typedef struct
@@ -906,6 +907,8 @@ parse_basebackup_options(List *options, basebackup_options *opt)
opt->compression = BACKUP_COMPRESSION_GZIP;
else if (strcmp(optval, "lz4") == 0)
opt->compression = BACKUP_COMPRESSION_LZ4;
+ else if (strcmp(optval, "zstd") == 0)
+ opt->compression = BACKUP_COMPRESSION_ZSTD;
else
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
@@ -1026,6 +1029,8 @@ SendBaseBackup(BaseBackupCmd *cmd)
sink = bbsink_gzip_new(sink, opt.compression_level);
else if (opt.compression == BACKUP_COMPRESSION_LZ4)
sink = bbsink_lz4_new(sink, opt.compression_level);
+ else if (opt.compression == BACKUP_COMPRESSION_ZSTD)
+ sink = bbsink_zstd_new(sink, opt.compression_level);
/* Set up progress reporting. */
sink = bbsink_progress_new(sink, opt.progress);
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
new file mode 100644
index 0000000000..24993a5bb6
--- /dev/null
+++ b/src/backend/replication/basebackup_zstd.c
@@ -0,0 +1,294 @@
+/*-------------------------------------------------------------------------
+ *
+ * basebackup_zstd.c
+ * Basebackup sink implementing zstd compression.
+ *
+ * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/basebackup_zstd.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#ifdef HAVE_LIBZSTD
+#include <zstd.h>
+#endif
+
+#include "replication/basebackup_sink.h"
+
+#ifdef HAVE_LIBZSTD
+
+typedef struct bbsink_zstd
+{
+ /* Common information for all types of sink. */
+ bbsink base;
+
+ /* Compression level */
+ int compresslevel;
+
+ ZSTD_CCtx *cctx;
+ ZSTD_outBuffer zstd_outBuf;
+} bbsink_zstd;
+
+static void bbsink_zstd_begin_backup(bbsink *sink);
+static void bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name);
+static void bbsink_zstd_archive_contents(bbsink *sink, size_t avail_in);
+static void bbsink_zstd_manifest_contents(bbsink *sink, size_t len);
+static void bbsink_zstd_end_archive(bbsink *sink);
+static void bbsink_zstd_cleanup(bbsink *sink);
+static void bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr,
+ TimeLineID endtli);
+
+const bbsink_ops bbsink_zstd_ops = {
+ .begin_backup = bbsink_zstd_begin_backup,
+ .begin_archive = bbsink_zstd_begin_archive,
+ .archive_contents = bbsink_zstd_archive_contents,
+ .end_archive = bbsink_zstd_end_archive,
+ .begin_manifest = bbsink_forward_begin_manifest,
+ .manifest_contents = bbsink_zstd_manifest_contents,
+ .end_manifest = bbsink_forward_end_manifest,
+ .end_backup = bbsink_zstd_end_backup,
+ .cleanup = bbsink_zstd_cleanup
+};
+#endif
+
+/*
+ * Create a new basebackup sink that performs zstd compression using the
+ * designated compression level.
+ */
+bbsink *
+bbsink_zstd_new(bbsink *next, int compresslevel)
+{
+#ifndef HAVE_LIBZSTD
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("zstd compression is not supported by this build")));
+ return NULL; /* keep compiler quiet */
+#else
+ bbsink_zstd *sink;
+
+ Assert(next != NULL);
+
+ if (compresslevel < 0 || compresslevel > 22)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("zstd compression level %d is out of range",
+ compresslevel)));
+
+ sink = palloc0(sizeof(bbsink_zstd));
+ *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
+ sink->base.bbs_next = next;
+ sink->compresslevel = compresslevel;
+
+ return &sink->base;
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+
+/*
+ * Begin backup.
+ */
+static void
+bbsink_zstd_begin_backup(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ size_t output_buffer_bound;
+
+ mysink->cctx = ZSTD_createCCtx();
+ if (!mysink->cctx)
+ elog(ERROR, "could not create zstd compression context");
+
+ ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
+ mysink->compresslevel);
+
+ /*
+ * We need our own buffer, because we're going to pass different data to
+ * the next sink than what gets passed to us.
+ */
+ mysink->base.bbs_buffer = palloc(mysink->base.bbs_buffer_length);
+
+ /*
+ * Make sure that the next sink's bbs_buffer is big enough to accommodate
+ * the compressed input buffer.
+ */
+ output_buffer_bound = ZSTD_compressBound(mysink->base.bbs_buffer_length);
+
+ /*
+ * The buffer length is expected to be a multiple of BLCKSZ, so round up.
+ */
+ output_buffer_bound = output_buffer_bound + BLCKSZ -
+ (output_buffer_bound % BLCKSZ);
+
+ bbsink_begin_backup(sink->bbs_next, sink->bbs_state, output_buffer_bound);
+}
+
+/*
+ * Prepare to compress the next archive.
+ */
+static void
+bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ char *zstd_archive_name;
+
+ /*
+ * At the start of each archive we reset the state to start a new
+ * compression operation. The parameters are sticky and they will stick
+ * around as we are resetting with option ZSTD_reset_session_only.
+ */
+ ZSTD_CCtx_reset(mysink->cctx, ZSTD_reset_session_only);
+
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+
+ /* Add ".zst" to the archive name. */
+ zstd_archive_name = psprintf("%s.zst", archive_name);
+ Assert(sink->bbs_next != NULL);
+ bbsink_begin_archive(sink->bbs_next, zstd_archive_name);
+ pfree(zstd_archive_name);
+}
+
+/*
+ * Compress the input data to the output buffer until we run out of input
+ * data. Each time the output buffer falls below the compression bound for
+ * the input buffer, invoke the archive_contents() method for the next sink.
+ *
+ * Note that since we're compressing the input, it may very commonly happen
+ * that we consume all the input data without filling the output buffer. In
+ * that case, the compressed representation of the current input data won't
+ * actually be sent to the next bbsink until a later call to this function,
+ * or perhaps even not until bbsink_zstd_end_archive() is invoked.
+ */
+static void
+bbsink_zstd_archive_contents(bbsink *sink, size_t len)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ ZSTD_inBuffer inBuf = {mysink->base.bbs_buffer, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t yet_to_flush;
+ size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+
+ /*
+ * If the out buffer is not left with enough space, send the output
+ * buffer to the next sink, and reset it.
+ */
+ if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mysink->cctx, &mysink->zstd_outBuf,
+ &inBuf, ZSTD_e_continue);
+
+ if (ZSTD_isError(yet_to_flush))
+ elog(ERROR, "could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ }
+}
+
+/*
+ * There might be some data inside zstd's internal buffers; we need to get that
+ * flushed out, also end the zstd frame and then get that forwarded to the
+ * successor sink as archive content.
+ *
+ * Then we can end processing for this archive.
+ */
+static void
+bbsink_zstd_end_archive(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ size_t yet_to_flush;
+
+ do
+ {
+ ZSTD_inBuffer in = {NULL, 0, 0};
+ size_t required_outBuf_bound = ZSTD_compressBound(0);
+
+ /*
+ * If the out buffer is not left with enough space, send the output
+ * buffer to the next sink, and reset it.
+ */
+ if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mysink->cctx,
+ &mysink->zstd_outBuf,
+ &in, ZSTD_e_end);
+
+ if (ZSTD_isError(yet_to_flush))
+ elog(ERROR, "could not compress data: %s",
+ ZSTD_getErrorName(yet_to_flush));
+
+ } while (yet_to_flush > 0);
+
+ /* Make sure to pass any remaining bytes to the next sink. */
+ if (mysink->zstd_outBuf.pos > 0)
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+
+ /* Pass on the information that this archive has ended. */
+ bbsink_forward_end_archive(sink);
+}
+
+/*
+ * Free the resources and context.
+ */
+static void
+bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr,
+ TimeLineID endtli)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+
+ /* Release the context. */
+ if (mysink->cctx)
+ {
+ ZSTD_freeCCtx(mysink->cctx);
+ mysink->cctx = NULL;
+ }
+
+ bbsink_forward_end_backup(sink, endptr, endtli);
+}
+
+/*
+ * Manifest contents are not compressed, but we do need to copy them into
+ * the successor sink's buffer, because we have our own.
+ */
+static void
+bbsink_zstd_manifest_contents(bbsink *sink, size_t len)
+{
+ memcpy(sink->bbs_next->bbs_buffer, sink->bbs_buffer, len);
+ bbsink_manifest_contents(sink->bbs_next, len);
+}
+
+/*
+ * In case the backup fails, make sure we free the compression context by
+ * calling ZSTD_freeCCtx if needed to avoid memory leak.
+ */
+static void
+bbsink_zstd_cleanup(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+
+ /* Release the context if not already released. */
+ if (mysink->cctx)
+ {
+ ZSTD_freeCCtx(mysink->cctx);
+ mysink->cctx = NULL;
+ }
+}
+
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 0003b59615..304d510220 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -391,8 +391,9 @@ usage(void)
printf(_(" -X, --wal-method=none|fetch|stream\n"
" include required WAL files with specified method\n"));
printf(_(" -z, --gzip compress tar output\n"));
- printf(_(" -Z, --compress={[{client,server}-]gzip,lz4,none}[:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n"
" compress tar output with given compression method or level\n"));
+ printf(_(" -Z, --compress=none do not compress tar output\n"));
printf(_("\nGeneral options:\n"));
printf(_(" -c, --checkpoint=fast|spread\n"
" set fast or spread checkpointing\n"));
@@ -1023,6 +1024,11 @@ parse_compress_options(char *src, WalCompressionMethod *methodres,
*methodres = COMPRESSION_LZ4;
*locationres = COMPRESS_LOCATION_SERVER;
}
+ else if (pg_strcasecmp(firstpart, "server-zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_SERVER;
+ }
else if (pg_strcasecmp(firstpart, "none") == 0)
{
*methodres = COMPRESSION_NONE;
@@ -1970,6 +1976,9 @@ BaseBackup(void)
case COMPRESSION_LZ4:
compressmethodstr = "lz4";
break;
+ case COMPRESSION_ZSTD:
+ compressmethodstr = "zstd";
+ break;
default:
Assert(false);
break;
@@ -2819,6 +2828,14 @@ main(int argc, char **argv)
exit(1);
}
break;
+ case COMPRESSION_ZSTD:
+ if (compresslevel > 22)
+ {
+ pg_log_error("compression level %d of method %s higher than maximum of 22",
+ compresslevel, "zstd");
+ exit(1);
+ }
+ break;
}
/*
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index ccb215c398..9b7656c692 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -904,6 +904,10 @@ main(int argc, char **argv)
exit(1);
#endif
break;
+ case COMPRESSION_ZSTD:
+ pg_log_error("compression with %s is not yet supported", "ZSTD");
+ exit(1);
+
}
diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h
index 2dfb353baa..ec54019cfc 100644
--- a/src/bin/pg_basebackup/walmethods.h
+++ b/src/bin/pg_basebackup/walmethods.h
@@ -24,6 +24,7 @@ typedef enum
{
COMPRESSION_GZIP,
COMPRESSION_LZ4,
+ COMPRESSION_ZSTD,
COMPRESSION_NONE
} WalCompressionMethod;
diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 851233a6e0..596df15118 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -10,6 +10,7 @@ export TAR
# name.
export GZIP_PROGRAM=$(GZIP)
export LZ4=$(LZ4)
+export ZSTD=$(ZSTD)
subdir = src/bin/pg_verifybackup
top_builddir = ../../..
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
old mode 100644
new mode 100755
index 6927ca4c74..1ccc6cb9df
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -43,6 +43,14 @@ my @test_configuration = (
'decompress_program' => $ENV{'LZ4'},
'decompress_flags' => [ '-d', '-m'],
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'server-zstd'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
@@ -108,6 +116,7 @@ for my $tc (@test_configuration)
# Cleanup.
unlink($backup_path . '/backup_manifest');
unlink($backup_path . '/base.tar');
+ unlink($backup_path . '/' . $tc->{'backup_archive'});
rmtree($extract_path);
}
}
diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h
index a3f8d37258..a7f16758a4 100644
--- a/src/include/replication/basebackup_sink.h
+++ b/src/include/replication/basebackup_sink.h
@@ -285,6 +285,7 @@ extern void bbsink_forward_cleanup(bbsink *sink);
extern bbsink *bbsink_copystream_new(bool send_to_client);
extern bbsink *bbsink_gzip_new(bbsink *next, int compresslevel);
extern bbsink *bbsink_lz4_new(bbsink *next, int compresslevel);
+extern bbsink *bbsink_zstd_new(bbsink *next, int compresslevel);
extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size);
extern bbsink *bbsink_server_new(bbsink *next, char *pathname);
extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate);
--
2.25.1
[application/octet-stream] v13-0003-ZSTD-add-client-side-compression-support.patch (11.9K, ../../CANm22Cim1JFzvbk+N7EynYRpovmhhAU5btxC=QUDDtzccnKjDQ@mail.gmail.com/6-v13-0003-ZSTD-add-client-side-compression-support.patch)
download | inline diff:
From 60408e8186d35f979071dc69ee556cc15c7f85b0 Mon Sep 17 00:00:00 2001
From: Jeevan Ladhe <[email protected]>
Date: Wed, 16 Feb 2022 22:22:27 +0530
Subject: [PATCH 3/4] ZSTD: add client-side compression support.
ZSTD compression can now be performed on the client using
pg_basebackup -Ft --compress client-zstd[:LEVEL].
Example:
pg_basebackup -D /tmp/zstd_client -Ft -Xnone --compress=client-zstd
---
src/bin/pg_basebackup/Makefile | 1 +
src/bin/pg_basebackup/bbstreamer.h | 2 +
src/bin/pg_basebackup/bbstreamer_zstd.c | 202 ++++++++++++++++++
src/bin/pg_basebackup/pg_basebackup.c | 28 ++-
src/bin/pg_verifybackup/t/010_client_untar.pl | 8 +
src/tools/msvc/Mkvcbuild.pm | 1 +
6 files changed, 240 insertions(+), 2 deletions(-)
create mode 100644 src/bin/pg_basebackup/bbstreamer_zstd.c
mode change 100644 => 100755 src/bin/pg_verifybackup/t/010_client_untar.pl
diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 1d0db4f9d0..0035ebcef5 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -44,6 +44,7 @@ BBOBJS = \
bbstreamer_gzip.o \
bbstreamer_inject.o \
bbstreamer_lz4.o \
+ bbstreamer_zstd.o \
bbstreamer_tar.o
all: pg_basebackup pg_receivewal pg_recvlogical
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
index c2de77bacc..bfc624a863 100644
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ b/src/bin/pg_basebackup/bbstreamer.h
@@ -209,6 +209,8 @@ extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
int compresslevel);
extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
+extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
+ int compresslevel);
extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
new file mode 100644
index 0000000000..0b20267cf4
--- /dev/null
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -0,0 +1,202 @@
+/*-------------------------------------------------------------------------
+ *
+ * bbstreamer_zstd.c
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/bin/pg_basebackup/bbstreamer_zstd.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include <unistd.h>
+
+#ifdef HAVE_LIBZSTD
+#include <zstd.h>
+#endif
+
+#include "bbstreamer.h"
+#include "common/logging.h"
+
+#ifdef HAVE_LIBZSTD
+
+typedef struct bbstreamer_zstd_frame
+{
+ bbstreamer base;
+
+ ZSTD_CCtx *cctx;
+ ZSTD_outBuffer zstd_outBuf;
+} bbstreamer_zstd_frame;
+
+static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context);
+static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
+static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+
+const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
+ .content = bbstreamer_zstd_compressor_content,
+ .finalize = bbstreamer_zstd_compressor_finalize,
+ .free = bbstreamer_zstd_compressor_free
+};
+#endif
+
+/*
+ * Create a new base backup streamer that performs zstd compression of tar
+ * blocks.
+ */
+bbstreamer *
+bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
+{
+#ifdef HAVE_LIBZSTD
+ bbstreamer_zstd_frame *streamer;
+
+ Assert(next != NULL);
+
+ streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+
+ *((const bbstreamer_ops **) &streamer->base.bbs_ops) =
+ &bbstreamer_zstd_compressor_ops;
+
+ streamer->base.bbs_next = next;
+ initStringInfo(&streamer->base.bbs_buffer);
+ enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize());
+
+ streamer->cctx = ZSTD_createCCtx();
+ if (!streamer->cctx)
+ pg_log_error("could not create zstd compression context");
+
+ /* Initialize stream compression preferences */
+ ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
+ compresslevel);
+
+ /* Initialize the ZSTD output buffer. */
+ streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
+ streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
+ streamer->zstd_outBuf.pos = 0;
+
+ return &streamer->base;
+#else
+ pg_log_error("this build does not support zstd compression");
+ exit(1);
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+/*
+ * Compress the input data to output buffer.
+ *
+ * Find out the compression bound based on input data length for each
+ * invocation to make sure that output buffer has enough capacity to
+ * accommodate the compressed data. In case if the output buffer
+ * capacity falls short of compression bound then forward the content
+ * of output buffer to next streamer and empty the buffer.
+ */
+static void
+bbstreamer_zstd_compressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ ZSTD_inBuffer inBuf = {data, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t yet_to_flush;
+ size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+
+ /*
+ * If the output buffer is not left with enough space, send the
+ * compressed bytes to the next streamer, and empty the buffer.
+ */
+ if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, member,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ context);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mystreamer->cctx, &mystreamer->zstd_outBuf,
+ &inBuf, ZSTD_e_continue);
+
+ if (ZSTD_isError(yet_to_flush))
+ pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ }
+}
+
+/*
+ * End-of-stream processing.
+ */
+static void
+bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ size_t yet_to_flush;
+
+ do
+ {
+ ZSTD_inBuffer in = {NULL, 0, 0};
+ size_t required_outBuf_bound = ZSTD_compressBound(0);
+
+ /*
+ * If the output buffer is not left with enough space, send the
+ * compressed bytes to the next streamer, and empty the buffer.
+ */
+ if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ BBSTREAMER_UNKNOWN);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mystreamer->cctx,
+ &mystreamer->zstd_outBuf,
+ &in, ZSTD_e_end);
+
+ if (ZSTD_isError(yet_to_flush))
+ pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+
+ } while (yet_to_flush > 0);
+
+ /* Make sure to pass any remaining bytes to the next streamer. */
+ if (mystreamer->zstd_outBuf.pos > 0)
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ BBSTREAMER_UNKNOWN);
+
+ bbstreamer_finalize(mystreamer->base.bbs_next);
+}
+
+/*
+ * Free memory.
+ */
+static void
+bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ bbstreamer_free(streamer->bbs_next);
+ ZSTD_freeCCtx(mystreamer->cctx);
+ pfree(streamer->bbs_buffer.data);
+ pfree(streamer);
+}
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 304d510220..18bd0df9a5 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -1024,6 +1024,16 @@ parse_compress_options(char *src, WalCompressionMethod *methodres,
*methodres = COMPRESSION_LZ4;
*locationres = COMPRESS_LOCATION_SERVER;
}
+ else if (pg_strcasecmp(firstpart, "zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_UNSPECIFIED;
+ }
+ else if (pg_strcasecmp(firstpart, "client-zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_CLIENT;
+ }
else if (pg_strcasecmp(firstpart, "server-zstd") == 0)
{
*methodres = COMPRESSION_ZSTD;
@@ -1147,7 +1157,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
bool inject_manifest;
bool is_tar,
is_tar_gz,
- is_tar_lz4;
+ is_tar_lz4,
+ is_tar_zstd;
bool must_parse_archive;
int archive_name_len = strlen(archive_name);
@@ -1170,6 +1181,10 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
is_tar_lz4 = (archive_name_len > 8 &&
strcmp(archive_name + archive_name_len - 4, ".lz4") == 0);
+ /* Is this a ZSTD archive? */
+ is_tar_zstd = (archive_name_len > 8 &&
+ strcmp(archive_name + archive_name_len - 4, ".zst") == 0);
+
/*
* We have to parse the archive if (1) we're suppose to extract it, or if
* (2) we need to inject backup_manifest or recovery configuration into it.
@@ -1179,7 +1194,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
(spclocation == NULL && writerecoveryconf));
/* At present, we only know how to parse tar archives. */
- if (must_parse_archive && !is_tar && !is_tar_gz && !is_tar_lz4)
+ if (must_parse_archive && !is_tar && !is_tar_gz && !is_tar_lz4
+ && !is_tar_zstd)
{
pg_log_error("unable to parse archive: %s", archive_name);
pg_log_info("only tar archives can be parsed");
@@ -1251,6 +1267,14 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
streamer = bbstreamer_lz4_compressor_new(streamer,
compresslevel);
}
+ else if (compressmethod == COMPRESSION_ZSTD)
+ {
+ strlcat(archive_filename, ".zst", sizeof(archive_filename));
+ streamer = bbstreamer_plain_writer_new(archive_filename,
+ archive_file);
+ streamer = bbstreamer_zstd_compressor_new(streamer,
+ compresslevel);
+ }
else
{
Assert(false); /* not reachable */
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
old mode 100644
new mode 100755
index 3616529390..c2a6161be6
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -42,6 +42,14 @@ my @test_configuration = (
'decompress_flags' => [ '-d' ],
'output_file' => 'base.tar',
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:5'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index bab81bd459..901e755d01 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -380,6 +380,7 @@ sub mkvcbuild
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_gzip.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_inject.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_lz4.c');
+ $pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_zstd.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_tar.c');
$pgbasebackup->AddLibrary('ws2_32.lib');
--
2.25.1
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-04 08:31 ` Dipesh Pandit <[email protected]>
2022-03-04 14:31 ` walmethods.c is kind of a mess (was Re: refactoring basebackup.c) Robert Haas <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Dipesh Pandit @ 2022-03-04 08:31 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Robert Haas <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Hi,
> > It will be good if we can also fix
> > CreateWalTarMethod to support LZ4 and ZSTD.
> Ok we will see, either Dipesh or I will take care of it.
I took a look at the CreateWalTarMethod to support LZ4 compression
for WAL files. The current implementation involves a 3 step to backup
a WAL file to a tar archive. For each file:
1. It first writes the header in the function tar_open_for_write,
flushes the contents of tar to disk and stores the header offset.
2. Next, the contents of WAL are written to the tar archive.
3. In the end, it recalculates the checksum in function tar_close() and
overwrites the header at an offset stored in step #1.
The need for overwriting header in CreateWalTarMethod is mainly related to
partial WAL files where the size of the WAL file < WalSegSize. The file is
being
padded and checksum is recalculated after adding pad bytes.
If we go ahead and implement LZ4 support for CreateWalTarMethod then
we have a problem here at step #3. In order to achieve better compression
ratio, compressed LZ4 blocks are linked to each other and these blocks
are decoded sequentially. If we overwrite the header as part of step #3 then
it corrupts the link between compressed LZ4 blocks. Although LZ4 provides
an option to write the compressed block independently (using blockMode
option set to LZ4F_blockIndepedent) but it is still a problem because we
don't
know if overwriting the header after recalculating the checksum will not
overlap
the boundary of the next block.
GZIP manages to overcome this problem as it provides an option to turn
on/off
compression on the fly while writing a compressed archive with the help of
zlib
library function deflateParams(). The current gzip implementation for
CreateWalTarMethod uses this library function to turn off compression just
before
step #1 and it writes the uncompressed header of size equal to
TAR_BLOCK_SIZE.
It uses the same library function to turn on the compression for writing
the contents
of the WAL file as part of step #2. It again turns off the compression just
before step
#3 to overwrite the header. The header is overwritten at the same offset
with size
equal to TAR_BLOCK_SIZE.
Since GZIP provides this option to enable/disable compression, it is
possible to
control the size of data we are writing to a compressed archive. Even if we
overwrite
an already written block in a compressed archive there is no risk of it
overlapping
with the boundary of the next block. This mechanism is not available in LZ4
and ZSTD.
In order to support LZ4 and ZSTD compression for CreateWalTarMethod we may
need to refactor this code unless I am missing something. We need to
somehow
add the padding bytes in case of partial WAL before we send it to the
compressed
archive. This will make sure that all files which are being compressed does
not
require any padding as the size is always equal to WalSegSize. There is no
need to
recalculate the checksum and we can avoid overwriting the header as part of
step #3.
Thoughts?
Thanks,
Dipesh
^ permalink raw reply [nested|flat] 103+ messages in thread
* walmethods.c is kind of a mess (was Re: refactoring basebackup.c)
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-04 08:31 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
@ 2022-03-04 14:31 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-04 14:31 UTC (permalink / raw)
To: Dipesh Pandit <[email protected]>; +Cc: pgsql-hackers; Michael Paquier <[email protected]>; Magnus Hagander <[email protected]>
On Fri, Mar 4, 2022 at 3:32 AM Dipesh Pandit <[email protected]> wrote:
> GZIP manages to overcome this problem as it provides an option to turn on/off
> compression on the fly while writing a compressed archive with the help of zlib
> library function deflateParams(). The current gzip implementation for
> CreateWalTarMethod uses this library function to turn off compression just before
> step #1 and it writes the uncompressed header of size equal to TAR_BLOCK_SIZE.
> It uses the same library function to turn on the compression for writing the contents
> of the WAL file as part of step #2. It again turns off the compression just before step
> #3 to overwrite the header. The header is overwritten at the same offset with size
> equal to TAR_BLOCK_SIZE.
This is a real mess. To me, it seems like a pretty big hack to use
deflateParams() to shut off compression in the middle of the
compressed data stream so that we can go back and overwrite that part
of the data later. It appears that the only reason we need that hack
is because we don't know the file size starting out. Except we kind of
do know the size, because pad_to_size specifies a minimum size for the
file. It's true that the maximum file size is unbounded, but I'm not
sure why that's important. I wonder if anyone else has an idea why we
didn't just set the file size to pad_to_size exactly when we write the
tar header the first time, instead of this IMHO kind of nutty approach
where we back up. I'd try to figure it out from the comments, but
there basically aren't any. I also had a look at the relevant commit
messages and didn't see anything relevant there either. If I'm missing
something, please point it out.
While I'm complaining, I noticed while looking at this code that it is
documented that "The caller must ensure that only one method is
instantiated in any given program, and that it's only instantiated
once!" As far as I can see, this is because somebody thought about
putting all of the relevant data into a struct and then decided on an
alternative strategy of storing some of it there, and the rest in a
global variable. I can't quite imagine why anyone would think that was
a good idea. There may be some reason that I can't see right now, but
here again there appear to be no relevant code comments.
I'm somewhat inclined to wonder whether we could just get rid of
walmethods.c entirely and use the new bbstreamer stuff instead. That
code also knows how to write plain files into a directory, and write
tar archives, and compress stuff, but in my totally biased opinion as
the author of most of that code, it's better code. It has no
restriction on using at most one method per program, or of
instantiating that method only once, and it already has LZ4 support,
and there's a pending patch for ZSTD support that I intend to get
committed soon as well. It also has, and I know I might be beating a
dead horse here, comments. Now, admittedly, it does need to know the
size of each archive member up front in order to work, so if we can't
solve the problem then we can't go this route. But if we can't solve
that problem, then we also can't add LZ4 and ZSTD support to
walmethods.c, because random access to compressed data is not really a
thing, even if we hacked it to work for gzip.
Thoughts?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-07 21:25 ` Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-07 21:25 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Wed, Feb 16, 2022 at 8:46 PM Jeevan Ladhe <[email protected]> wrote:
> Thanks for the comments Robert. I have addressed your comments in the
> attached patch v13-0002-ZSTD-add-server-side-compression-support.patch.
> Rest of the patches are similar to v12, but just bumped the version number.
OK, here's a consolidated patch with all your changes from 0002-0004
as 0001 plus a few proposed edits of my own in 0002. By and large I
think this is fine.
My proposed changes are largely cosmetic, but one thing that isn't is
revising the size - pos <= bound tests to instead check size - pos <
bound. My reasoning for that change is: if the number of bytes
remaining in the buffer is exactly equal to the maximum number we can
write, we don't need to flush it yet. If that sounds correct, we
should fix the LZ4 code the same way.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v14-0002-My-changes.patch (7.6K, ../../CA+Tgmobyzfbz=gyze2_LL1ZumZunmaEKbHQxjrFkOR7APZGu-g@mail.gmail.com/2-v14-0002-My-changes.patch)
download | inline diff:
From 76a910744597ab95cabbbfc68872832f18289aa1 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Mon, 7 Mar 2022 16:20:32 -0500
Subject: [PATCH v14 2/2] My changes.
---
doc/src/sgml/ref/pg_basebackup.sgml | 7 ++---
src/backend/replication/basebackup_zstd.c | 33 +++++++++++++----------
src/bin/pg_basebackup/bbstreamer_zstd.c | 23 +++++++++-------
3 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 4cf28a2a61..4a630b59b7 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -431,7 +431,7 @@ PostgreSQL documentation
When the tar format is used with <literal>gzip</literal>,
<literal>lz4</literal>, or <literal>zstd</literal>, the suffix
<filename>.gz</filename>, <filename>.lz4</filename>, or
- <filename>.zst</filename> respectively will be automatically added to
+ <filename>.zst</filename>, respectively, will be automatically added to
all tar filenames. When the plain format is used, client-side
compression may not be specified, but it is still possible to request
server-side compression. If this is done, the server will compress the
@@ -441,8 +441,9 @@ PostgreSQL documentation
When this option is used in combination with
<literal>-Xstream</literal>, <literal>pg_wal.tar</literal> will
be compressed using <literal>gzip</literal> if client-side gzip
- compression is selected, but will not be compressed if server-side
- compression, LZ4, or ZSTD compression is selected.
+ compression is selected, but will not be compressed if any other
+ compression algorithm is selected, or if server-side compression
+ is selected.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index 24993a5bb6..e3f9b1d4dc 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -172,18 +172,19 @@ bbsink_zstd_archive_contents(bbsink *sink, size_t len)
while (inBuf.pos < inBuf.size)
{
size_t yet_to_flush;
- size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+ size_t max_needed = ZSTD_compressBound(inBuf.size - inBuf.pos);
/*
* If the out buffer is not left with enough space, send the output
* buffer to the next sink, and reset it.
*/
- if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
- required_outBuf_bound)
+ if (mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos < max_needed)
{
- bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ bbsink_archive_contents(mysink->base.bbs_next,
+ mysink->zstd_outBuf.pos);
mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
- mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.size =
+ mysink->base.bbs_next->bbs_buffer_length;
mysink->zstd_outBuf.pos = 0;
}
@@ -191,7 +192,9 @@ bbsink_zstd_archive_contents(bbsink *sink, size_t len)
&inBuf, ZSTD_e_continue);
if (ZSTD_isError(yet_to_flush))
- elog(ERROR, "could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ elog(ERROR,
+ "could not compress data: %s",
+ ZSTD_getErrorName(yet_to_flush));
}
}
@@ -211,18 +214,19 @@ bbsink_zstd_end_archive(bbsink *sink)
do
{
ZSTD_inBuffer in = {NULL, 0, 0};
- size_t required_outBuf_bound = ZSTD_compressBound(0);
+ size_t max_needed = ZSTD_compressBound(0);
/*
* If the out buffer is not left with enough space, send the output
* buffer to the next sink, and reset it.
*/
- if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
- required_outBuf_bound)
+ if (mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos < max_needed)
{
- bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ bbsink_archive_contents(mysink->base.bbs_next,
+ mysink->zstd_outBuf.pos);
mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
- mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.size =
+ mysink->base.bbs_next->bbs_buffer_length;
mysink->zstd_outBuf.pos = 0;
}
@@ -238,7 +242,8 @@ bbsink_zstd_end_archive(bbsink *sink)
/* Make sure to pass any remaining bytes to the next sink. */
if (mysink->zstd_outBuf.pos > 0)
- bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ bbsink_archive_contents(mysink->base.bbs_next,
+ mysink->zstd_outBuf.pos);
/* Pass on the information that this archive has ended. */
bbsink_forward_end_archive(sink);
@@ -275,8 +280,8 @@ bbsink_zstd_manifest_contents(bbsink *sink, size_t len)
}
/*
- * In case the backup fails, make sure we free the compression context by
- * calling ZSTD_freeCCtx if needed to avoid memory leak.
+ * In case the backup fails, make sure we free any compression context that
+ * got allocated, so that we don't leak memory.
*/
static void
bbsink_zstd_cleanup(bbsink *sink)
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index 83b59d63ba..cc68367dd5 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -121,14 +121,14 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
while (inBuf.pos < inBuf.size)
{
size_t yet_to_flush;
- size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+ size_t max_needed = ZSTD_compressBound(inBuf.size - inBuf.pos);
/*
* If the output buffer is not left with enough space, send the
* compressed bytes to the next streamer, and empty the buffer.
*/
- if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
- required_outBuf_bound)
+ if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
+ max_needed)
{
bbstreamer_content(mystreamer->base.bbs_next, member,
mystreamer->zstd_outBuf.dst,
@@ -141,11 +141,13 @@ bbstreamer_zstd_compressor_content(bbstreamer *streamer,
mystreamer->zstd_outBuf.pos = 0;
}
- yet_to_flush = ZSTD_compressStream2(mystreamer->cctx, &mystreamer->zstd_outBuf,
- &inBuf, ZSTD_e_continue);
+ yet_to_flush =
+ ZSTD_compressStream2(mystreamer->cctx, &mystreamer->zstd_outBuf,
+ &inBuf, ZSTD_e_continue);
if (ZSTD_isError(yet_to_flush))
- pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ pg_log_error("could not compress data: %s",
+ ZSTD_getErrorName(yet_to_flush));
}
}
@@ -161,14 +163,14 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
do
{
ZSTD_inBuffer in = {NULL, 0, 0};
- size_t required_outBuf_bound = ZSTD_compressBound(0);
+ size_t max_needed = ZSTD_compressBound(0);
/*
* If the output buffer is not left with enough space, send the
* compressed bytes to the next streamer, and empty the buffer.
*/
- if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
- required_outBuf_bound)
+ if (mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos <
+ max_needed)
{
bbstreamer_content(mystreamer->base.bbs_next, NULL,
mystreamer->zstd_outBuf.dst,
@@ -186,7 +188,8 @@ bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
&in, ZSTD_e_end);
if (ZSTD_isError(yet_to_flush))
- pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ pg_log_error("could not compress data: %s",
+ ZSTD_getErrorName(yet_to_flush));
} while (yet_to_flush > 0);
--
2.24.3 (Apple Git-128)
[application/octet-stream] v14-0001-Patches-from-JL.patch (35.9K, ../../CA+Tgmobyzfbz=gyze2_LL1ZumZunmaEKbHQxjrFkOR7APZGu-g@mail.gmail.com/3-v14-0001-Patches-from-JL.patch)
download | inline diff:
From b22f46c645a999ec7bab702bb418a6c5e6bf234e Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Mon, 7 Mar 2022 15:08:45 -0500
Subject: [PATCH v14 1/2] Patches from JL.
---
doc/src/sgml/protocol.sgml | 7 +-
doc/src/sgml/ref/pg_basebackup.sgml | 38 +-
src/backend/replication/Makefile | 1 +
src/backend/replication/basebackup.c | 7 +-
src/backend/replication/basebackup_zstd.c | 294 +++++++++++++++
src/bin/pg_basebackup/Makefile | 1 +
src/bin/pg_basebackup/bbstreamer.h | 3 +
src/bin/pg_basebackup/bbstreamer_zstd.c | 335 ++++++++++++++++++
src/bin/pg_basebackup/pg_basebackup.c | 49 ++-
src/bin/pg_basebackup/pg_receivewal.c | 4 +
src/bin/pg_basebackup/walmethods.h | 1 +
src/bin/pg_verifybackup/Makefile | 1 +
src/bin/pg_verifybackup/t/008_untar.pl | 9 +
src/bin/pg_verifybackup/t/009_extract.pl | 5 +
src/bin/pg_verifybackup/t/010_client_untar.pl | 8 +
src/include/replication/basebackup_sink.h | 1 +
src/tools/msvc/Mkvcbuild.pm | 1 +
17 files changed, 740 insertions(+), 25 deletions(-)
create mode 100644 src/backend/replication/basebackup_zstd.c
create mode 100644 src/bin/pg_basebackup/bbstreamer_zstd.c
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c51c4254a7..0695bcd423 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2724,8 +2724,8 @@ The commands accepted in replication mode are:
<listitem>
<para>
Instructs the server to compress the backup using the specified
- method. Currently, the supported methods are <literal>gzip</literal>
- and <literal>lz4</literal>.
+ method. Currently, the supported methods are <literal>gzip</literal>,
+ <literal>lz4</literal>, and <literal>zstd</literal>.
</para>
</listitem>
</varlistentry>
@@ -2737,7 +2737,8 @@ The commands accepted in replication mode are:
Specifies the compression level to be used. This should only be
used in conjunction with the <literal>COMPRESSION</literal> option.
For <literal>gzip</literal> the value should be an integer between 1
- and 9, and for <literal>lz4</literal> it should be between 1 and 12.
+ and 9, for <literal>lz4</literal> between 1 and 12, and for
+ <literal>zstd</literal> it should be between 1 and 22.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 53aa40dcd1..4cf28a2a61 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -417,30 +417,32 @@ PostgreSQL documentation
specify <literal>-Xfetch</literal>.
</para>
<para>
- The compression method can be set to <literal>gzip</literal> or
- <literal>lz4</literal>, or <literal>none</literal> for no
- compression. A compression level can be optionally specified, by
- appending the level number after a colon (<literal>:</literal>). If no
- level is specified, the default compression level will be used. If
- only a level is specified without mentioning an algorithm,
- <literal>gzip</literal> compression will be used if the level is
- greater than 0, and no compression will be used if the level is 0.
- </para>
- <para>
- When the tar format is used with <literal>gzip</literal> or
- <literal>lz4</literal>, the suffix <filename>.gz</filename> or
- <filename>.lz4</filename> will automatically be added to all tar
- filenames. When the plain format is used, client-side compression may
- not be specified, but it is still possible to request server-side
- compression. If this is done, the server will compress the backup for
- transmission, and the client will decompress and extract it.
+ The compression method can be set to <literal>gzip</literal>,
+ <literal>lz4</literal>, <literal>zstd</literal>, or
+ <literal>none</literal> for no compression. A compression level can
+ optionally be specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default
+ compression level will be used. If only a level is specified without
+ mentioning an algorithm, <literal>gzip</literal> compression will be
+ used if the level is greater than 0, and no compression will be used if
+ the level is 0.
+ </para>
+ <para>
+ When the tar format is used with <literal>gzip</literal>,
+ <literal>lz4</literal>, or <literal>zstd</literal>, the suffix
+ <filename>.gz</filename>, <filename>.lz4</filename>, or
+ <filename>.zst</filename> respectively will be automatically added to
+ all tar filenames. When the plain format is used, client-side
+ compression may not be specified, but it is still possible to request
+ server-side compression. If this is done, the server will compress the
+ backup for transmission, and the client will decompress and extract it.
</para>
<para>
When this option is used in combination with
<literal>-Xstream</literal>, <literal>pg_wal.tar</literal> will
be compressed using <literal>gzip</literal> if client-side gzip
compression is selected, but will not be compressed if server-side
- compresion or LZ4 compresion is selected.
+ compression, LZ4, or ZSTD compression is selected.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile
index 74043ff331..2e6de7007f 100644
--- a/src/backend/replication/Makefile
+++ b/src/backend/replication/Makefile
@@ -20,6 +20,7 @@ OBJS = \
basebackup_copy.o \
basebackup_gzip.o \
basebackup_lz4.o \
+ basebackup_zstd.o \
basebackup_progress.o \
basebackup_server.o \
basebackup_sink.o \
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 0bf28b55d7..2378ce5c5e 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -64,7 +64,8 @@ typedef enum
{
BACKUP_COMPRESSION_NONE,
BACKUP_COMPRESSION_GZIP,
- BACKUP_COMPRESSION_LZ4
+ BACKUP_COMPRESSION_LZ4,
+ BACKUP_COMPRESSION_ZSTD
} basebackup_compression_type;
typedef struct
@@ -906,6 +907,8 @@ parse_basebackup_options(List *options, basebackup_options *opt)
opt->compression = BACKUP_COMPRESSION_GZIP;
else if (strcmp(optval, "lz4") == 0)
opt->compression = BACKUP_COMPRESSION_LZ4;
+ else if (strcmp(optval, "zstd") == 0)
+ opt->compression = BACKUP_COMPRESSION_ZSTD;
else
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
@@ -1026,6 +1029,8 @@ SendBaseBackup(BaseBackupCmd *cmd)
sink = bbsink_gzip_new(sink, opt.compression_level);
else if (opt.compression == BACKUP_COMPRESSION_LZ4)
sink = bbsink_lz4_new(sink, opt.compression_level);
+ else if (opt.compression == BACKUP_COMPRESSION_ZSTD)
+ sink = bbsink_zstd_new(sink, opt.compression_level);
/* Set up progress reporting. */
sink = bbsink_progress_new(sink, opt.progress);
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
new file mode 100644
index 0000000000..24993a5bb6
--- /dev/null
+++ b/src/backend/replication/basebackup_zstd.c
@@ -0,0 +1,294 @@
+/*-------------------------------------------------------------------------
+ *
+ * basebackup_zstd.c
+ * Basebackup sink implementing zstd compression.
+ *
+ * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/replication/basebackup_zstd.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#ifdef HAVE_LIBZSTD
+#include <zstd.h>
+#endif
+
+#include "replication/basebackup_sink.h"
+
+#ifdef HAVE_LIBZSTD
+
+typedef struct bbsink_zstd
+{
+ /* Common information for all types of sink. */
+ bbsink base;
+
+ /* Compression level */
+ int compresslevel;
+
+ ZSTD_CCtx *cctx;
+ ZSTD_outBuffer zstd_outBuf;
+} bbsink_zstd;
+
+static void bbsink_zstd_begin_backup(bbsink *sink);
+static void bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name);
+static void bbsink_zstd_archive_contents(bbsink *sink, size_t avail_in);
+static void bbsink_zstd_manifest_contents(bbsink *sink, size_t len);
+static void bbsink_zstd_end_archive(bbsink *sink);
+static void bbsink_zstd_cleanup(bbsink *sink);
+static void bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr,
+ TimeLineID endtli);
+
+const bbsink_ops bbsink_zstd_ops = {
+ .begin_backup = bbsink_zstd_begin_backup,
+ .begin_archive = bbsink_zstd_begin_archive,
+ .archive_contents = bbsink_zstd_archive_contents,
+ .end_archive = bbsink_zstd_end_archive,
+ .begin_manifest = bbsink_forward_begin_manifest,
+ .manifest_contents = bbsink_zstd_manifest_contents,
+ .end_manifest = bbsink_forward_end_manifest,
+ .end_backup = bbsink_zstd_end_backup,
+ .cleanup = bbsink_zstd_cleanup
+};
+#endif
+
+/*
+ * Create a new basebackup sink that performs zstd compression using the
+ * designated compression level.
+ */
+bbsink *
+bbsink_zstd_new(bbsink *next, int compresslevel)
+{
+#ifndef HAVE_LIBZSTD
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("zstd compression is not supported by this build")));
+ return NULL; /* keep compiler quiet */
+#else
+ bbsink_zstd *sink;
+
+ Assert(next != NULL);
+
+ if (compresslevel < 0 || compresslevel > 22)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("zstd compression level %d is out of range",
+ compresslevel)));
+
+ sink = palloc0(sizeof(bbsink_zstd));
+ *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
+ sink->base.bbs_next = next;
+ sink->compresslevel = compresslevel;
+
+ return &sink->base;
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+
+/*
+ * Begin backup.
+ */
+static void
+bbsink_zstd_begin_backup(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ size_t output_buffer_bound;
+
+ mysink->cctx = ZSTD_createCCtx();
+ if (!mysink->cctx)
+ elog(ERROR, "could not create zstd compression context");
+
+ ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
+ mysink->compresslevel);
+
+ /*
+ * We need our own buffer, because we're going to pass different data to
+ * the next sink than what gets passed to us.
+ */
+ mysink->base.bbs_buffer = palloc(mysink->base.bbs_buffer_length);
+
+ /*
+ * Make sure that the next sink's bbs_buffer is big enough to accommodate
+ * the compressed input buffer.
+ */
+ output_buffer_bound = ZSTD_compressBound(mysink->base.bbs_buffer_length);
+
+ /*
+ * The buffer length is expected to be a multiple of BLCKSZ, so round up.
+ */
+ output_buffer_bound = output_buffer_bound + BLCKSZ -
+ (output_buffer_bound % BLCKSZ);
+
+ bbsink_begin_backup(sink->bbs_next, sink->bbs_state, output_buffer_bound);
+}
+
+/*
+ * Prepare to compress the next archive.
+ */
+static void
+bbsink_zstd_begin_archive(bbsink *sink, const char *archive_name)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ char *zstd_archive_name;
+
+ /*
+ * At the start of each archive we reset the state to start a new
+ * compression operation. The parameters are sticky and they will stick
+ * around as we are resetting with option ZSTD_reset_session_only.
+ */
+ ZSTD_CCtx_reset(mysink->cctx, ZSTD_reset_session_only);
+
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+
+ /* Add ".zst" to the archive name. */
+ zstd_archive_name = psprintf("%s.zst", archive_name);
+ Assert(sink->bbs_next != NULL);
+ bbsink_begin_archive(sink->bbs_next, zstd_archive_name);
+ pfree(zstd_archive_name);
+}
+
+/*
+ * Compress the input data to the output buffer until we run out of input
+ * data. Each time the output buffer falls below the compression bound for
+ * the input buffer, invoke the archive_contents() method for the next sink.
+ *
+ * Note that since we're compressing the input, it may very commonly happen
+ * that we consume all the input data without filling the output buffer. In
+ * that case, the compressed representation of the current input data won't
+ * actually be sent to the next bbsink until a later call to this function,
+ * or perhaps even not until bbsink_zstd_end_archive() is invoked.
+ */
+static void
+bbsink_zstd_archive_contents(bbsink *sink, size_t len)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ ZSTD_inBuffer inBuf = {mysink->base.bbs_buffer, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t yet_to_flush;
+ size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+
+ /*
+ * If the out buffer is not left with enough space, send the output
+ * buffer to the next sink, and reset it.
+ */
+ if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mysink->cctx, &mysink->zstd_outBuf,
+ &inBuf, ZSTD_e_continue);
+
+ if (ZSTD_isError(yet_to_flush))
+ elog(ERROR, "could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ }
+}
+
+/*
+ * There might be some data inside zstd's internal buffers; we need to get that
+ * flushed out, also end the zstd frame and then get that forwarded to the
+ * successor sink as archive content.
+ *
+ * Then we can end processing for this archive.
+ */
+static void
+bbsink_zstd_end_archive(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+ size_t yet_to_flush;
+
+ do
+ {
+ ZSTD_inBuffer in = {NULL, 0, 0};
+ size_t required_outBuf_bound = ZSTD_compressBound(0);
+
+ /*
+ * If the out buffer is not left with enough space, send the output
+ * buffer to the next sink, and reset it.
+ */
+ if ((mysink->zstd_outBuf.size - mysink->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+ mysink->zstd_outBuf.dst = mysink->base.bbs_next->bbs_buffer;
+ mysink->zstd_outBuf.size = mysink->base.bbs_next->bbs_buffer_length;
+ mysink->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mysink->cctx,
+ &mysink->zstd_outBuf,
+ &in, ZSTD_e_end);
+
+ if (ZSTD_isError(yet_to_flush))
+ elog(ERROR, "could not compress data: %s",
+ ZSTD_getErrorName(yet_to_flush));
+
+ } while (yet_to_flush > 0);
+
+ /* Make sure to pass any remaining bytes to the next sink. */
+ if (mysink->zstd_outBuf.pos > 0)
+ bbsink_archive_contents(mysink->base.bbs_next, mysink->zstd_outBuf.pos);
+
+ /* Pass on the information that this archive has ended. */
+ bbsink_forward_end_archive(sink);
+}
+
+/*
+ * Free the resources and context.
+ */
+static void
+bbsink_zstd_end_backup(bbsink *sink, XLogRecPtr endptr,
+ TimeLineID endtli)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+
+ /* Release the context. */
+ if (mysink->cctx)
+ {
+ ZSTD_freeCCtx(mysink->cctx);
+ mysink->cctx = NULL;
+ }
+
+ bbsink_forward_end_backup(sink, endptr, endtli);
+}
+
+/*
+ * Manifest contents are not compressed, but we do need to copy them into
+ * the successor sink's buffer, because we have our own.
+ */
+static void
+bbsink_zstd_manifest_contents(bbsink *sink, size_t len)
+{
+ memcpy(sink->bbs_next->bbs_buffer, sink->bbs_buffer, len);
+ bbsink_manifest_contents(sink->bbs_next, len);
+}
+
+/*
+ * In case the backup fails, make sure we free the compression context by
+ * calling ZSTD_freeCCtx if needed to avoid memory leak.
+ */
+static void
+bbsink_zstd_cleanup(bbsink *sink)
+{
+ bbsink_zstd *mysink = (bbsink_zstd *) sink;
+
+ /* Release the context if not already released. */
+ if (mysink->cctx)
+ {
+ ZSTD_freeCCtx(mysink->cctx);
+ mysink->cctx = NULL;
+ }
+}
+
+#endif
diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 1d0db4f9d0..0035ebcef5 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -44,6 +44,7 @@ BBOBJS = \
bbstreamer_gzip.o \
bbstreamer_inject.o \
bbstreamer_lz4.o \
+ bbstreamer_zstd.o \
bbstreamer_tar.o
all: pg_basebackup pg_receivewal pg_recvlogical
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
index c2de77bacc..02d4c05df6 100644
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ b/src/bin/pg_basebackup/bbstreamer.h
@@ -209,6 +209,9 @@ extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
int compresslevel);
extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
+extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
+ int compresslevel);
+extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
new file mode 100644
index 0000000000..83b59d63ba
--- /dev/null
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -0,0 +1,335 @@
+/*-------------------------------------------------------------------------
+ *
+ * bbstreamer_zstd.c
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/bin/pg_basebackup/bbstreamer_zstd.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include <unistd.h>
+
+#ifdef HAVE_LIBZSTD
+#include <zstd.h>
+#endif
+
+#include "bbstreamer.h"
+#include "common/logging.h"
+
+#ifdef HAVE_LIBZSTD
+
+typedef struct bbstreamer_zstd_frame
+{
+ bbstreamer base;
+
+ ZSTD_CCtx *cctx;
+ ZSTD_DCtx *dctx;
+ ZSTD_outBuffer zstd_outBuf;
+} bbstreamer_zstd_frame;
+
+static void bbstreamer_zstd_compressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context);
+static void bbstreamer_zstd_compressor_finalize(bbstreamer *streamer);
+static void bbstreamer_zstd_compressor_free(bbstreamer *streamer);
+
+const bbstreamer_ops bbstreamer_zstd_compressor_ops = {
+ .content = bbstreamer_zstd_compressor_content,
+ .finalize = bbstreamer_zstd_compressor_finalize,
+ .free = bbstreamer_zstd_compressor_free
+};
+
+static void bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context);
+static void bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer);
+static void bbstreamer_zstd_decompressor_free(bbstreamer *streamer);
+
+const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
+ .content = bbstreamer_zstd_decompressor_content,
+ .finalize = bbstreamer_zstd_decompressor_finalize,
+ .free = bbstreamer_zstd_decompressor_free
+};
+#endif
+
+/*
+ * Create a new base backup streamer that performs zstd compression of tar
+ * blocks.
+ */
+bbstreamer *
+bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
+{
+#ifdef HAVE_LIBZSTD
+ bbstreamer_zstd_frame *streamer;
+
+ Assert(next != NULL);
+
+ streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+
+ *((const bbstreamer_ops **) &streamer->base.bbs_ops) =
+ &bbstreamer_zstd_compressor_ops;
+
+ streamer->base.bbs_next = next;
+ initStringInfo(&streamer->base.bbs_buffer);
+ enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize());
+
+ streamer->cctx = ZSTD_createCCtx();
+ if (!streamer->cctx)
+ pg_log_error("could not create zstd compression context");
+
+ /* Initialize stream compression preferences */
+ ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
+ compresslevel);
+
+ /* Initialize the ZSTD output buffer. */
+ streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
+ streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
+ streamer->zstd_outBuf.pos = 0;
+
+ return &streamer->base;
+#else
+ pg_log_error("this build does not support zstd compression");
+ exit(1);
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+/*
+ * Compress the input data to output buffer.
+ *
+ * Find out the compression bound based on input data length for each
+ * invocation to make sure that output buffer has enough capacity to
+ * accommodate the compressed data. In case if the output buffer
+ * capacity falls short of compression bound then forward the content
+ * of output buffer to next streamer and empty the buffer.
+ */
+static void
+bbstreamer_zstd_compressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ ZSTD_inBuffer inBuf = {data, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t yet_to_flush;
+ size_t required_outBuf_bound = ZSTD_compressBound(inBuf.size - inBuf.pos);
+
+ /*
+ * If the output buffer is not left with enough space, send the
+ * compressed bytes to the next streamer, and empty the buffer.
+ */
+ if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, member,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ context);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mystreamer->cctx, &mystreamer->zstd_outBuf,
+ &inBuf, ZSTD_e_continue);
+
+ if (ZSTD_isError(yet_to_flush))
+ pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+ }
+}
+
+/*
+ * End-of-stream processing.
+ */
+static void
+bbstreamer_zstd_compressor_finalize(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ size_t yet_to_flush;
+
+ do
+ {
+ ZSTD_inBuffer in = {NULL, 0, 0};
+ size_t required_outBuf_bound = ZSTD_compressBound(0);
+
+ /*
+ * If the output buffer is not left with enough space, send the
+ * compressed bytes to the next streamer, and empty the buffer.
+ */
+ if ((mystreamer->zstd_outBuf.size - mystreamer->zstd_outBuf.pos) <=
+ required_outBuf_bound)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ BBSTREAMER_UNKNOWN);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ yet_to_flush = ZSTD_compressStream2(mystreamer->cctx,
+ &mystreamer->zstd_outBuf,
+ &in, ZSTD_e_end);
+
+ if (ZSTD_isError(yet_to_flush))
+ pg_log_error("could not compress data: %s", ZSTD_getErrorName(yet_to_flush));
+
+ } while (yet_to_flush > 0);
+
+ /* Make sure to pass any remaining bytes to the next streamer. */
+ if (mystreamer->zstd_outBuf.pos > 0)
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ BBSTREAMER_UNKNOWN);
+
+ bbstreamer_finalize(mystreamer->base.bbs_next);
+}
+
+/*
+ * Free memory.
+ */
+static void
+bbstreamer_zstd_compressor_free(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ bbstreamer_free(streamer->bbs_next);
+ ZSTD_freeCCtx(mystreamer->cctx);
+ pfree(streamer->bbs_buffer.data);
+ pfree(streamer);
+}
+#endif
+
+/*
+ * Create a new base backup streamer that performs decompression of zstd
+ * compressed blocks.
+ */
+bbstreamer *
+bbstreamer_zstd_decompressor_new(bbstreamer *next)
+{
+#ifdef HAVE_LIBZSTD
+ bbstreamer_zstd_frame *streamer;
+
+ Assert(next != NULL);
+
+ streamer = palloc0(sizeof(bbstreamer_zstd_frame));
+ *((const bbstreamer_ops **) &streamer->base.bbs_ops) =
+ &bbstreamer_zstd_decompressor_ops;
+
+ streamer->base.bbs_next = next;
+ initStringInfo(&streamer->base.bbs_buffer);
+ enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize());
+
+ streamer->dctx = ZSTD_createDCtx();
+ if (!streamer->dctx)
+ {
+ pg_log_error("could not create zstd decompression context");
+ exit(1);
+ }
+
+ /* Initialize the ZSTD output buffer. */
+ streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
+ streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
+ streamer->zstd_outBuf.pos = 0;
+
+ return &streamer->base;
+#else
+ pg_log_error("this build does not support compression");
+ exit(1);
+#endif
+}
+
+#ifdef HAVE_LIBZSTD
+/*
+ * Decompress the input data to output buffer until we run out of input
+ * data. Each time the output buffer is full, pass on the decompressed data
+ * to the next streamer.
+ */
+static void
+bbstreamer_zstd_decompressor_content(bbstreamer *streamer,
+ bbstreamer_member *member,
+ const char *data, int len,
+ bbstreamer_archive_context context)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+ ZSTD_inBuffer inBuf = {data, len, 0};
+
+ while (inBuf.pos < inBuf.size)
+ {
+ size_t ret;
+
+ /*
+ * If output buffer is full then forward the content to next streamer
+ * and update the output buffer.
+ */
+ if (mystreamer->zstd_outBuf.pos >= mystreamer->zstd_outBuf.size)
+ {
+ bbstreamer_content(mystreamer->base.bbs_next, member,
+ mystreamer->zstd_outBuf.dst,
+ mystreamer->zstd_outBuf.pos,
+ context);
+
+ /* Reset the ZSTD output buffer. */
+ mystreamer->zstd_outBuf.dst = mystreamer->base.bbs_buffer.data;
+ mystreamer->zstd_outBuf.size = mystreamer->base.bbs_buffer.maxlen;
+ mystreamer->zstd_outBuf.pos = 0;
+ }
+
+ ret = ZSTD_decompressStream(mystreamer->dctx,
+ &mystreamer->zstd_outBuf, &inBuf);
+
+ if (ZSTD_isError(ret))
+ pg_log_error("could not decompress data: %s", ZSTD_getErrorName(ret));
+ }
+}
+
+/*
+ * End-of-stream processing.
+ */
+static void
+bbstreamer_zstd_decompressor_finalize(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ /*
+ * End of the stream, if there is some pending data in output buffers then
+ * we must forward it to next streamer.
+ */
+ if (mystreamer->zstd_outBuf.pos > 0)
+ bbstreamer_content(mystreamer->base.bbs_next, NULL,
+ mystreamer->base.bbs_buffer.data,
+ mystreamer->base.bbs_buffer.maxlen,
+ BBSTREAMER_UNKNOWN);
+
+ bbstreamer_finalize(mystreamer->base.bbs_next);
+}
+
+/*
+ * Free memory.
+ */
+static void
+bbstreamer_zstd_decompressor_free(bbstreamer *streamer)
+{
+ bbstreamer_zstd_frame *mystreamer = (bbstreamer_zstd_frame *) streamer;
+
+ bbstreamer_free(streamer->bbs_next);
+ ZSTD_freeDCtx(mystreamer->dctx);
+ pfree(streamer->bbs_buffer.data);
+ pfree(streamer);
+}
+#endif
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index c1ed7aeeee..9f3ecc60fb 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -405,8 +405,9 @@ usage(void)
printf(_(" -X, --wal-method=none|fetch|stream\n"
" include required WAL files with specified method\n"));
printf(_(" -z, --gzip compress tar output\n"));
- printf(_(" -Z, --compress={[{client,server}-]gzip,lz4,none}[:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n"
" compress tar output with given compression method or level\n"));
+ printf(_(" -Z, --compress=none do not compress tar output\n"));
printf(_("\nGeneral options:\n"));
printf(_(" -c, --checkpoint=fast|spread\n"
" set fast or spread checkpointing\n"));
@@ -1067,6 +1068,21 @@ parse_compress_options(char *src, WalCompressionMethod *methodres,
*methodres = COMPRESSION_LZ4;
*locationres = COMPRESS_LOCATION_SERVER;
}
+ else if (pg_strcasecmp(firstpart, "zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_UNSPECIFIED;
+ }
+ else if (pg_strcasecmp(firstpart, "client-zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_CLIENT;
+ }
+ else if (pg_strcasecmp(firstpart, "server-zstd") == 0)
+ {
+ *methodres = COMPRESSION_ZSTD;
+ *locationres = COMPRESS_LOCATION_SERVER;
+ }
else if (pg_strcasecmp(firstpart, "none") == 0)
{
*methodres = COMPRESSION_NONE;
@@ -1191,7 +1207,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
bool inject_manifest;
bool is_tar,
is_tar_gz,
- is_tar_lz4;
+ is_tar_lz4,
+ is_tar_zstd;
bool must_parse_archive;
int archive_name_len = strlen(archive_name);
@@ -1214,6 +1231,10 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
is_tar_lz4 = (archive_name_len > 8 &&
strcmp(archive_name + archive_name_len - 4, ".lz4") == 0);
+ /* Is this a ZSTD archive? */
+ is_tar_zstd = (archive_name_len > 8 &&
+ strcmp(archive_name + archive_name_len - 4, ".zst") == 0);
+
/*
* We have to parse the archive if (1) we're suppose to extract it, or if
* (2) we need to inject backup_manifest or recovery configuration into it.
@@ -1223,7 +1244,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
(spclocation == NULL && writerecoveryconf));
/* At present, we only know how to parse tar archives. */
- if (must_parse_archive && !is_tar && !is_tar_gz && !is_tar_lz4)
+ if (must_parse_archive && !is_tar && !is_tar_gz && !is_tar_lz4
+ && !is_tar_zstd)
{
pg_log_error("unable to parse archive: %s", archive_name);
pg_log_info("only tar archives can be parsed");
@@ -1295,6 +1317,14 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
streamer = bbstreamer_lz4_compressor_new(streamer,
compresslevel);
}
+ else if (compressmethod == COMPRESSION_ZSTD)
+ {
+ strlcat(archive_filename, ".zst", sizeof(archive_filename));
+ streamer = bbstreamer_plain_writer_new(archive_filename,
+ archive_file);
+ streamer = bbstreamer_zstd_compressor_new(streamer,
+ compresslevel);
+ }
else
{
Assert(false); /* not reachable */
@@ -1353,6 +1383,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
streamer = bbstreamer_gzip_decompressor_new(streamer);
else if (compressmethod == COMPRESSION_LZ4)
streamer = bbstreamer_lz4_decompressor_new(streamer);
+ else if (compressmethod == COMPRESSION_ZSTD)
+ streamer = bbstreamer_zstd_decompressor_new(streamer);
}
/* Return the results. */
@@ -2020,6 +2052,9 @@ BaseBackup(void)
case COMPRESSION_LZ4:
compressmethodstr = "lz4";
break;
+ case COMPRESSION_ZSTD:
+ compressmethodstr = "zstd";
+ break;
default:
Assert(false);
break;
@@ -2869,6 +2904,14 @@ main(int argc, char **argv)
exit(1);
}
break;
+ case COMPRESSION_ZSTD:
+ if (compresslevel > 22)
+ {
+ pg_log_error("compression level %d of method %s higher than maximum of 22",
+ compresslevel, "zstd");
+ exit(1);
+ }
+ break;
}
/*
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index ce661a9ce4..8a4c2b8964 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -904,6 +904,10 @@ main(int argc, char **argv)
exit(1);
#endif
break;
+ case COMPRESSION_ZSTD:
+ pg_log_error("compression with %s is not yet supported", "ZSTD");
+ exit(1);
+
}
diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h
index 2dfb353baa..ec54019cfc 100644
--- a/src/bin/pg_basebackup/walmethods.h
+++ b/src/bin/pg_basebackup/walmethods.h
@@ -24,6 +24,7 @@ typedef enum
{
COMPRESSION_GZIP,
COMPRESSION_LZ4,
+ COMPRESSION_ZSTD,
COMPRESSION_NONE
} WalCompressionMethod;
diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 851233a6e0..596df15118 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -10,6 +10,7 @@ export TAR
# name.
export GZIP_PROGRAM=$(GZIP)
export LZ4=$(LZ4)
+export ZSTD=$(ZSTD)
subdir = src/bin/pg_verifybackup
top_builddir = ../../..
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index 383203d0b8..efbc910dfb 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -42,6 +42,14 @@ my @test_configuration = (
'decompress_program' => $ENV{'LZ4'},
'decompress_flags' => [ '-d', '-m'],
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'server-zstd'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
@@ -107,6 +115,7 @@ for my $tc (@test_configuration)
# Cleanup.
unlink($backup_path . '/backup_manifest');
unlink($backup_path . '/base.tar');
+ unlink($backup_path . '/' . $tc->{'backup_archive'});
rmtree($extract_path);
}
}
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index c51cdf79f8..d30ba01742 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -31,6 +31,11 @@ my @test_configuration = (
'compression_method' => 'lz4',
'backup_flags' => ['--compress', 'server-lz4:5'],
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'server-zstd:5'],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 3616529390..c2a6161be6 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -42,6 +42,14 @@ my @test_configuration = (
'decompress_flags' => [ '-d' ],
'output_file' => 'base.tar',
'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:5'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h
index a3f8d37258..a7f16758a4 100644
--- a/src/include/replication/basebackup_sink.h
+++ b/src/include/replication/basebackup_sink.h
@@ -285,6 +285,7 @@ extern void bbsink_forward_cleanup(bbsink *sink);
extern bbsink *bbsink_copystream_new(bool send_to_client);
extern bbsink *bbsink_gzip_new(bbsink *next, int compresslevel);
extern bbsink *bbsink_lz4_new(bbsink *next, int compresslevel);
+extern bbsink *bbsink_zstd_new(bbsink *next, int compresslevel);
extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size);
extern bbsink *bbsink_server_new(bbsink *next, char *pathname);
extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate);
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 105f5c72a2..441d6ae6bf 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -380,6 +380,7 @@ sub mkvcbuild
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_gzip.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_inject.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_lz4.c');
+ $pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_zstd.c');
$pgbasebackup->AddFile('src/bin/pg_basebackup/bbstreamer_tar.c');
$pgbasebackup->AddLibrary('ws2_32.lib');
--
2.24.3 (Apple Git-128)
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-08 09:49 ` Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jeevan Ladhe @ 2022-03-08 09:49 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Hi Robert,
My proposed changes are largely cosmetic, but one thing that isn't is
> revising the size - pos <= bound tests to instead check size - pos <
> bound. My reasoning for that change is: if the number of bytes
> remaining in the buffer is exactly equal to the maximum number we can
> write, we don't need to flush it yet. If that sounds correct, we
> should fix the LZ4 code the same way.
>
I agree with your patch. The patch looks good to me.
Yes, the LZ4 flush check should also be fixed. Please find the attached
patch to fix the LZ4 code.
Regards,
Jeevan Ladhe
Attachments:
[application/octet-stream] fix_lz4_flush_logic.patch (2.5K, ../../CANm22CgVMa85O1akgs+DOPE8NSrT1zbz5_vYfS83_r+6nCivLQ@mail.gmail.com/3-fix_lz4_flush_logic.patch)
download | inline diff:
diff --git a/src/backend/replication/basebackup_lz4.c b/src/backend/replication/basebackup_lz4.c
index d26032783c..472b620d7c 100644
--- a/src/backend/replication/basebackup_lz4.c
+++ b/src/backend/replication/basebackup_lz4.c
@@ -193,7 +193,7 @@ bbsink_lz4_archive_contents(bbsink *sink, size_t avail_in)
* LZ4F_compressBound(), ask the next sink to process the data so that we
* can empty the buffer.
*/
- if ((mysink->base.bbs_next->bbs_buffer_length - mysink->bytes_written) <=
+ if ((mysink->base.bbs_next->bbs_buffer_length - mysink->bytes_written) <
avail_in_bound)
{
bbsink_archive_contents(sink->bbs_next, mysink->bytes_written);
@@ -238,7 +238,7 @@ bbsink_lz4_end_archive(bbsink *sink)
Assert(mysink->base.bbs_next->bbs_buffer_length >= lz4_footer_bound);
- if ((mysink->base.bbs_next->bbs_buffer_length - mysink->bytes_written) <=
+ if ((mysink->base.bbs_next->bbs_buffer_length - mysink->bytes_written) <
lz4_footer_bound)
{
bbsink_archive_contents(sink->bbs_next, mysink->bytes_written);
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c
index f0bc226bf8..bde018246f 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/bbstreamer_lz4.c
@@ -99,7 +99,7 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, int compresslevel)
compressed_bound = LZ4F_compressBound(streamer->base.bbs_buffer.maxlen, prefs);
/* Enlarge buffer if it falls short of compression bound. */
- if (streamer->base.bbs_buffer.maxlen <= compressed_bound)
+ if (streamer->base.bbs_buffer.maxlen < compressed_bound)
enlargeStringInfo(&streamer->base.bbs_buffer, compressed_bound);
ctxError = LZ4F_createCompressionContext(&streamer->cctx, LZ4F_VERSION);
@@ -170,7 +170,7 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer,
*/
out_bound = LZ4F_compressBound(len, &mystreamer->prefs);
Assert(mystreamer->base.bbs_buffer.maxlen >= out_bound);
- if (avail_out <= out_bound)
+ if (avail_out < out_bound)
{
bbstreamer_content(mystreamer->base.bbs_next, member,
mystreamer->base.bbs_buffer.data,
@@ -218,7 +218,7 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer)
/* Find out the footer bound and update the output buffer. */
footer_bound = LZ4F_compressBound(0, &mystreamer->prefs);
Assert(mystreamer->base.bbs_buffer.maxlen >= footer_bound);
- if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <=
+ if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) <
footer_bound)
{
bbstreamer_content(mystreamer->base.bbs_next, NULL,
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-08 15:28 ` Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-08 15:28 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Tue, Mar 8, 2022 at 4:49 AM Jeevan Ladhe <[email protected]> wrote:
> I agree with your patch. The patch looks good to me.
> Yes, the LZ4 flush check should also be fixed. Please find the attached
> patch to fix the LZ4 code.
OK, committed all that stuff.
I think we also need to fix one other thing. Right now, for LZ4
support we test HAVE_LIBLZ4, but TOAST and XLOG compression are
testing USE_LZ4, so I think we should be doing the same here. And
similarly I think we should be testing USE_ZSTD not HAVE_LIBZSTD.
Patch for that attached.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] fix-symbol-tests.patch (11.3K, ../../CA+Tgmoap+hTD2-QNPJLH4tffeFE8MX5+xkbFKMU3FKBy=ZSNKA@mail.gmail.com/2-fix-symbol-tests.patch)
download | inline diff:
diff --git a/src/backend/replication/basebackup_lz4.c b/src/backend/replication/basebackup_lz4.c
index 472b620d7c..d838f723d0 100644
--- a/src/backend/replication/basebackup_lz4.c
+++ b/src/backend/replication/basebackup_lz4.c
@@ -12,13 +12,13 @@
*/
#include "postgres.h"
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
#include <lz4frame.h>
#endif
#include "replication/basebackup_sink.h"
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
typedef struct bbsink_lz4
{
@@ -62,7 +62,7 @@ const bbsink_ops bbsink_lz4_ops = {
bbsink *
bbsink_lz4_new(bbsink *next, int compresslevel)
{
-#ifndef HAVE_LIBLZ4
+#ifndef USE_LZ4
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("lz4 compression is not supported by this build")));
@@ -87,7 +87,7 @@ bbsink_lz4_new(bbsink *next, int compresslevel)
#endif
}
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
/*
* Begin backup.
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index e3f9b1d4dc..c0e2be6e27 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -12,13 +12,13 @@
*/
#include "postgres.h"
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
#include <zstd.h>
#endif
#include "replication/basebackup_sink.h"
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
typedef struct bbsink_zstd
{
@@ -61,7 +61,7 @@ const bbsink_ops bbsink_zstd_ops = {
bbsink *
bbsink_zstd_new(bbsink *next, int compresslevel)
{
-#ifndef HAVE_LIBZSTD
+#ifndef USE_ZSTD
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("zstd compression is not supported by this build")));
@@ -86,7 +86,7 @@ bbsink_zstd_new(bbsink *next, int compresslevel)
#endif
}
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
/*
* Begin backup.
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c
index bde018246f..810052e4e3 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/bbstreamer_lz4.c
@@ -13,7 +13,7 @@
#include <unistd.h>
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
#include <lz4frame.h>
#endif
@@ -22,7 +22,7 @@
#include "common/file_perm.h"
#include "common/string.h"
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
typedef struct bbstreamer_lz4_frame
{
bbstreamer base;
@@ -69,7 +69,7 @@ const bbstreamer_ops bbstreamer_lz4_decompressor_ops = {
bbstreamer *
bbstreamer_lz4_compressor_new(bbstreamer *next, int compresslevel)
{
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
bbstreamer_lz4_frame *streamer;
LZ4F_errorCode_t ctxError;
LZ4F_preferences_t *prefs;
@@ -114,7 +114,7 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, int compresslevel)
#endif
}
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
/*
* Compress the input data to output buffer.
*
@@ -280,7 +280,7 @@ bbstreamer_lz4_compressor_free(bbstreamer *streamer)
bbstreamer *
bbstreamer_lz4_decompressor_new(bbstreamer *next)
{
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
bbstreamer_lz4_frame *streamer;
LZ4F_errorCode_t ctxError;
@@ -309,7 +309,7 @@ bbstreamer_lz4_decompressor_new(bbstreamer *next)
#endif
}
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
/*
* Decompress the input data to output buffer until we run out of input
* data. Each time the output buffer is full, pass on the decompressed data
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index cc68367dd5..e86749a8fb 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -13,14 +13,14 @@
#include <unistd.h>
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
#include <zstd.h>
#endif
#include "bbstreamer.h"
#include "common/logging.h"
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
typedef struct bbstreamer_zstd_frame
{
@@ -65,7 +65,7 @@ const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
bbstreamer *
bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
{
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
bbstreamer_zstd_frame *streamer;
Assert(next != NULL);
@@ -99,7 +99,7 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
#endif
}
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
/*
* Compress the input data to output buffer.
*
@@ -225,7 +225,7 @@ bbstreamer_zstd_compressor_free(bbstreamer *streamer)
bbstreamer *
bbstreamer_zstd_decompressor_new(bbstreamer *next)
{
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
bbstreamer_zstd_frame *streamer;
Assert(next != NULL);
@@ -257,7 +257,7 @@ bbstreamer_zstd_decompressor_new(bbstreamer *next)
#endif
}
-#ifdef HAVE_LIBZSTD
+#ifdef USE_ZSTD
/*
* Decompress the input data to output buffer until we run out of input
* data. Each time the output buffer is full, pass on the decompressed data
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 8a4c2b8964..e2ceafeb0f 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -32,7 +32,7 @@
#include "receivelog.h"
#include "streamutil.h"
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
#include "lz4frame.h"
#endif
@@ -382,7 +382,7 @@ FindStreamingStart(uint32 *tli)
}
else if (!ispartial && wal_compression_method == COMPRESSION_LZ4)
{
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
#define LZ4_CHUNK_SZ 64 * 1024 /* 64kB as maximum chunk size read */
int fd;
ssize_t r;
@@ -889,7 +889,7 @@ main(int argc, char **argv)
#endif
break;
case COMPRESSION_LZ4:
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (compresslevel != 0)
{
pg_log_error("cannot use --compress with --compression-method=%s",
diff --git a/src/bin/pg_basebackup/t/020_pg_receivewal.pl b/src/bin/pg_basebackup/t/020_pg_receivewal.pl
index 545618e0b2..8c38816b22 100644
--- a/src/bin/pg_basebackup/t/020_pg_receivewal.pl
+++ b/src/bin/pg_basebackup/t/020_pg_receivewal.pl
@@ -141,7 +141,7 @@ SKIP:
SKIP:
{
skip "postgres was not built with LZ4 support", 5
- if (!check_pg_config("#define HAVE_LIBLZ4 1"));
+ if (!check_pg_config("#define USE_LZ4 1"));
# Generate more WAL including one completed, compressed segment.
$primary->psql('postgres', 'SELECT pg_switch_wal();');
diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c
index a6d08c1270..1e0ff760eb 100644
--- a/src/bin/pg_basebackup/walmethods.c
+++ b/src/bin/pg_basebackup/walmethods.c
@@ -18,7 +18,7 @@
#include <time.h>
#include <unistd.h>
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
#include <lz4frame.h>
#endif
#ifdef HAVE_LIBZ
@@ -70,7 +70,7 @@ typedef struct DirectoryMethodFile
#ifdef HAVE_LIBZ
gzFile gzfp;
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
LZ4F_compressionContext_t ctx;
size_t lz4bufsize;
void *lz4buf;
@@ -114,7 +114,7 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_
#ifdef HAVE_LIBZ
gzFile gzfp = NULL;
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
LZ4F_compressionContext_t ctx = NULL;
size_t lz4bufsize = 0;
void *lz4buf = NULL;
@@ -160,7 +160,7 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_
}
}
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
size_t ctx_out;
@@ -245,7 +245,7 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_
gzclose(gzfp);
else
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
(void) LZ4F_compressEnd(ctx, lz4buf, lz4bufsize, NULL);
@@ -265,7 +265,7 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_
if (dir_data->compression_method == COMPRESSION_GZIP)
f->gzfp = gzfp;
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
f->ctx = ctx;
@@ -306,7 +306,7 @@ dir_write(Walfile f, const void *buf, size_t count)
}
else
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
size_t chunk;
@@ -394,7 +394,7 @@ dir_close(Walfile f, WalCloseMethod method)
}
else
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
size_t compressed;
@@ -487,7 +487,7 @@ dir_close(Walfile f, WalCloseMethod method)
if (r != 0)
dir_data->lasterrno = errno;
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
pg_free(df->lz4buf);
/* supports free on NULL */
LZ4F_freeCompressionContext(df->ctx);
@@ -523,7 +523,7 @@ dir_sync(Walfile f)
}
}
#endif
-#ifdef HAVE_LIBLZ4
+#ifdef USE_LZ4
if (dir_data->compression_method == COMPRESSION_LZ4)
{
DirectoryMethodFile *df = (DirectoryMethodFile *) f;
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 3e55ff26f8..fd1052e5db 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -3743,7 +3743,7 @@ if ($collation_check_stderr !~ /ERROR: /)
}
# Determine whether build supports LZ4.
-my $supports_lz4 = check_pg_config("#define HAVE_LIBLZ4 1");
+my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
# Create additional databases for mutations of schema public
$node->psql('postgres', 'create database regress_pg_dump_test;');
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index efbc910dfb..9bfd7023fb 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -41,7 +41,7 @@ my @test_configuration = (
'backup_archive' => 'base.tar.lz4',
'decompress_program' => $ENV{'LZ4'},
'decompress_flags' => [ '-d', '-m'],
- 'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ 'enabled' => check_pg_config("#define USE_LZ4 1")
},
{
'compression_method' => 'zstd',
@@ -49,7 +49,7 @@ my @test_configuration = (
'backup_archive' => 'base.tar.zst',
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
- 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
+ 'enabled' => check_pg_config("#define USE_ZSTD 1")
}
);
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index d30ba01742..9f9cc7540b 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -30,12 +30,12 @@ my @test_configuration = (
{
'compression_method' => 'lz4',
'backup_flags' => ['--compress', 'server-lz4:5'],
- 'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ 'enabled' => check_pg_config("#define USE_LZ4 1")
},
{
'compression_method' => 'zstd',
'backup_flags' => ['--compress', 'server-zstd:5'],
- 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
+ 'enabled' => check_pg_config("#define USE_ZSTD 1")
}
);
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index c2a6161be6..16a752195e 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -41,7 +41,7 @@ my @test_configuration = (
'decompress_program' => $ENV{'LZ4'},
'decompress_flags' => [ '-d' ],
'output_file' => 'base.tar',
- 'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+ 'enabled' => check_pg_config("#define USE_LZ4 1")
},
{
'compression_method' => 'zstd',
@@ -49,7 +49,7 @@ my @test_configuration = (
'backup_archive' => 'base.tar.zst',
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
- 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
+ 'enabled' => check_pg_config("#define USE_ZSTD 1")
}
);
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-08 16:32 ` Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jeevan Ladhe @ 2022-03-08 16:32 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
>
> OK, committed all that stuff.
>
Thanks for the commit Robert.
> I think we also need to fix one other thing. Right now, for LZ4
> support we test HAVE_LIBLZ4, but TOAST and XLOG compression are
> testing USE_LZ4, so I think we should be doing the same here. And
> similarly I think we should be testing USE_ZSTD not HAVE_LIBZSTD.
>
I reviewed the patch, and it seems to be capturing and replacing all the
places of HAVE_LIB* with USE_* correctly.
Just curious, apart from consistency, do you see other problems as well
when testing one vs the other?
Regards,
Jeevan Ladhe
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-08 16:53 ` Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-08 16:53 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>; Andres Freund <[email protected]>
On Tue, Mar 8, 2022 at 11:32 AM Jeevan Ladhe <[email protected]> wrote:
> I reviewed the patch, and it seems to be capturing and replacing all the
> places of HAVE_LIB* with USE_* correctly.
> Just curious, apart from consistency, do you see other problems as well
> when testing one vs the other?
So, the kind of problem you would worry about in a case like this is:
suppose that configure detects LIBLZ4, but the user specifies
--without-lz4. Then maybe there is some way for HAVE_LIBLZ4 to be
true, while USE_LIBLZ4 is false, and therefore we should not be
compiling code that uses LZ4 but do anyway. As configure.ac is
currently coded, I think that's impossible, because we only search for
liblz4 if the user says --with-lz4, and if they do that, then USE_LZ4
will be set. Therefore, I don't think there is a live problem here,
just an inconsistency.
Probably still best to clean it up before an angry Andres chases me
down, since I know he's working on the build system...
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-08 16:58 ` Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jeevan Ladhe @ 2022-03-08 16:58 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Justin Pryzby <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>; Andres Freund <[email protected]>
ok got it. Thanks for your insights.
Regards,
Jeevan Ladhe
On Tue, 8 Mar 2022 at 22:23, Robert Haas <[email protected]> wrote:
> On Tue, Mar 8, 2022 at 11:32 AM Jeevan Ladhe <[email protected]>
> wrote:
> > I reviewed the patch, and it seems to be capturing and replacing all the
> > places of HAVE_LIB* with USE_* correctly.
> > Just curious, apart from consistency, do you see other problems as well
> > when testing one vs the other?
>
> So, the kind of problem you would worry about in a case like this is:
> suppose that configure detects LIBLZ4, but the user specifies
> --without-lz4. Then maybe there is some way for HAVE_LIBLZ4 to be
> true, while USE_LIBLZ4 is false, and therefore we should not be
> compiling code that uses LZ4 but do anyway. As configure.ac is
> currently coded, I think that's impossible, because we only search for
> liblz4 if the user says --with-lz4, and if they do that, then USE_LZ4
> will be set. Therefore, I don't think there is a live problem here,
> just an inconsistency.
>
> Probably still best to clean it up before an angry Andres chases me
> down, since I know he's working on the build system...
>
> --
> Robert Haas
> EDB: http://www.enterprisedb.com
>
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-11 01:02 ` Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2022-03-11 01:02 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Robert Haas <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>; Andres Freund <[email protected]>
I'm getting errors from pg_basebackup when using both -D- and --compress=server-*
The issue seems to go away if I use --no-manifest.
$ ./src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method none --compress=server-gzip >/dev/null ; echo $?
pg_basebackup: error: tar member has empty name
1
$ ./src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method none --compress=server-gzip >/dev/null ; echo $?
NOTICE: WAL archiving is not enabled; you must ensure that all required WAL segments are copied through other means to complete the backup
pg_basebackup: error: COPY stream ended before last file was finished
1
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
@ 2022-03-11 15:19 ` Robert Haas <[email protected]>
2022-03-11 16:29 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
0 siblings, 2 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-11 15:19 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>; Andres Freund <[email protected]>
On Thu, Mar 10, 2022 at 8:02 PM Justin Pryzby <[email protected]> wrote:
> I'm getting errors from pg_basebackup when using both -D- and --compress=server-*
> The issue seems to go away if I use --no-manifest.
>
> $ ./src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method none --compress=server-gzip >/dev/null ; echo $?
> pg_basebackup: error: tar member has empty name
> 1
>
> $ ./src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method none --compress=server-gzip >/dev/null ; echo $?
> NOTICE: WAL archiving is not enabled; you must ensure that all required WAL segments are copied through other means to complete the backup
> pg_basebackup: error: COPY stream ended before last file was finished
> 1
Thanks for the report. The problem here is that, when the output is
standard output (-D -), pg_basebackup can only produce a single output
file, so the manifest gets injected into the tar file on the client
side rather than being written separately as we do in normal cases.
However, that only works if we're receiving a tar file that we can
parse from the server, and here the server is sending a compressed
tarfile. The current code mistakely attempts to parse the compressed
tarfile as if it were an uncompressed tarfile, which causes the error
messages that you are seeing (and which I can also reproduce here). We
actually have enough infrastructure available in pg_basebackup now
that we could do the "right thing" in this case: decompress the data
received from the server, parse the resulting tar file, inject the
backup manifest, construct a new tar file, and recompress. However, I
think that's probably not a good idea, because it's unlikely that the
user will understand that the data is being compressed on the server,
then decompressed, and then recompressed again, and the performance of
the resulting pipeline will probably not be very good. So I think we
should just refuse this command. Patch for that attached.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] reject-compressed-inject.patch (2.0K, ../../CA+Tgmob6Rnjz-Qv32h3yJn8nnUkLhrtQDAS4y5AtsgtorAFHRA@mail.gmail.com/2-reject-compressed-inject.patch)
download | inline diff:
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 9f3ecc60fb..43c4036eee 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -1208,7 +1208,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
bool is_tar,
is_tar_gz,
is_tar_lz4,
- is_tar_zstd;
+ is_tar_zstd,
+ is_compressed_tar;
bool must_parse_archive;
int archive_name_len = strlen(archive_name);
@@ -1235,6 +1236,24 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
is_tar_zstd = (archive_name_len > 8 &&
strcmp(archive_name + archive_name_len - 4, ".zst") == 0);
+ /* Is this any kind of compressed tar? */
+ is_compressed_tar = is_tar_gz || is_tar_lz4 || is_tar_zstd;
+
+ /*
+ * Injecting the manifest into a compressed tar file would be possible if
+ * we decompressed it, parsed the tarfile, generated a new tarfile, and
+ * recompressed it, but compressing and decompressing multiple times just
+ * to inject the manifest seems inefficient enough that it's probably not
+ * what the user wants. So, instead, reject the request and tell the user
+ * to specify something more reasonable.
+ */
+ if (inject_manifest && is_compressed_tar)
+ {
+ pg_log_error("cannot inject manifest into a compressed tarfile");
+ pg_log_info("use client-side compression, send the output to a directory rather than standard output, or use --no-manifest");
+ exit(1);
+ }
+
/*
* We have to parse the archive if (1) we're suppose to extract it, or if
* (2) we need to inject backup_manifest or recovery configuration into it.
@@ -1244,8 +1263,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
(spclocation == NULL && writerecoveryconf));
/* At present, we only know how to parse tar archives. */
- if (must_parse_archive && !is_tar && !is_tar_gz && !is_tar_lz4
- && !is_tar_zstd)
+ if (must_parse_archive && !is_tar && !is_compressed_tar)
{
pg_log_error("unable to parse archive: %s", archive_name);
pg_log_info("only tar archives can be parsed");
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-11 16:29 ` Justin Pryzby <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Justin Pryzby @ 2022-03-11 16:29 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>; Andres Freund <[email protected]>
On Fri, Mar 11, 2022 at 10:19:29AM -0500, Robert Haas wrote:
> So I think we should just refuse this command. Patch for that attached.
Sounds right.
Also, I think the magic 8 for .gz should actually be a 7.
I'm not sure why it tests for ".gz" but not ".tar.gz", which would help to make
them all less magic.
commit 1fb1e21ba7a500bb2b85ec3e65f59130fcdb4a7e
Author: Justin Pryzby <[email protected]>
Date: Thu Mar 10 21:22:16 2022 -0600
pg_basebackup: make magic numbers less magic
The magic 8 for .gz should actually be a 7.
.tar.gz
1234567
.tar.lz4
.tar.zst
12345678
See d45099425, 751b8d23b, 7cf085f07.
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 9f3ecc60fbe..8dd9721323d 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -1223,17 +1223,17 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
is_tar = (archive_name_len > 4 &&
strcmp(archive_name + archive_name_len - 4, ".tar") == 0);
- /* Is this a gzip archive? */
- is_tar_gz = (archive_name_len > 8 &&
- strcmp(archive_name + archive_name_len - 3, ".gz") == 0);
+ /* Is this a .tar.gz archive? */
+ is_tar_gz = (archive_name_len > 7 &&
+ strcmp(archive_name + archive_name_len - 7, ".tar.gz") == 0);
- /* Is this a LZ4 archive? */
+ /* Is this a .tar.lz4 archive? */
is_tar_lz4 = (archive_name_len > 8 &&
- strcmp(archive_name + archive_name_len - 4, ".lz4") == 0);
+ strcmp(archive_name + archive_name_len - 8, ".tar.lz4") == 0);
- /* Is this a ZSTD archive? */
+ /* Is this a .tar.zst archive? */
is_tar_zstd = (archive_name_len > 8 &&
- strcmp(archive_name + archive_name_len - 4, ".zst") == 0);
+ strcmp(archive_name + archive_name_len - 8, ".tar.zst") == 0);
/*
* We have to parse the archive if (1) we're suppose to extract it, or if
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-12 01:52 ` Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Andres Freund @ 2022-03-12 01:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Hi,
On 2022-03-11 10:19:29 -0500, Robert Haas wrote:
> Thanks for the report. The problem here is that, when the output is
> standard output (-D -), pg_basebackup can only produce a single output
> file, so the manifest gets injected into the tar file on the client
> side rather than being written separately as we do in normal cases.
> However, that only works if we're receiving a tar file that we can
> parse from the server, and here the server is sending a compressed
> tarfile. The current code mistakely attempts to parse the compressed
> tarfile as if it were an uncompressed tarfile, which causes the error
> messages that you are seeing (and which I can also reproduce here). We
> actually have enough infrastructure available in pg_basebackup now
> that we could do the "right thing" in this case: decompress the data
> received from the server, parse the resulting tar file, inject the
> backup manifest, construct a new tar file, and recompress. However, I
> think that's probably not a good idea, because it's unlikely that the
> user will understand that the data is being compressed on the server,
> then decompressed, and then recompressed again, and the performance of
> the resulting pipeline will probably not be very good. So I think we
> should just refuse this command. Patch for that attached.
You could also just append a manifest as a compresed tar to the compressed tar
stream. Unfortunately GNU tar requires -i to read concated compressed
archives, so perhaps that's not quite an alternative.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
@ 2022-03-14 13:27 ` Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-14 13:27 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Dipesh Pandit <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Fri, Mar 11, 2022 at 8:52 PM Andres Freund <[email protected]> wrote:
> You could also just append a manifest as a compresed tar to the compressed tar
> stream. Unfortunately GNU tar requires -i to read concated compressed
> archives, so perhaps that's not quite an alternative.
s/Unfortunately/Fortunately/ :-p
I think we've already gone way too far in the direction of making this
stuff rely on specific details of the tar format. What if someday we
wanted to switch to pax, cpio, zip, 7zip, whatever, or even just have
one of those things as an option? It's not that I'm dying to have
PostgreSQL produce rar or arj files, but I think we box ourselves into
a corner when we just assume tar everywhere. As an example of a
similar issue with real consequences, consider the recent discovery
that we can't easily add support for LZ4 or ZSTD compression of
pg_wal.tar. The problem is that the existing code tells the gzip
library to emit the tar header as part of the compressed stream
without actually compressing it, and then it goes back and overwrites
that data later! Unsurprisingly, that's not a feature every
compression library offers.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2022-03-14 16:11 ` Dipesh Pandit <[email protected]>
2022-03-14 16:35 ` Re: refactoring basebackup.c (zstd workers) Justin Pryzby <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
0 siblings, 3 replies; 103+ messages in thread
From: Dipesh Pandit @ 2022-03-14 16:11 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Hi,
I tried to implement support for parallel ZSTD compression. The
library provides an option (ZSTD_c_nbWorkers) to specify the
number of compression workers. The number of parallel
workers can be set as part of compression parameter and if this
option is specified then the library performs parallel compression
based on the specified number of workers.
User can specify the number of parallel worker as part of
--compress option by appending an integer value after at sign (@).
(-Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL][@WORKERS])
Please find the attached patch v1 with the above changes.
Note: ZSTD library version 1.5.x supports parallel compression
by default and if the library version is lower than 1.5.x then
parallel compression is enabled only the source is compiled with build
macro ZSTD_MULTITHREAD. If the linked library version doesn't
support parallel compression then setting the value of parameter
ZSTD_c_nbWorkers to a value other than 0 will be no-op and
returns an error.
Thanks,
Dipesh
Attachments:
[text/x-patch] v1-0001-support-parallel-zstd-compression.patch (16.0K, ../../CAN1g5_F0KBgitEVLO8T0ynaBcQBfRMCjH9wKguDJZpX1hF3gFA@mail.gmail.com/3-v1-0001-support-parallel-zstd-compression.patch)
download | inline diff:
From 688ad1e3f9b43bf911e8c3837497a874e4a6937f Mon Sep 17 00:00:00 2001
From: Dipesh Pandit <[email protected]>
Date: Mon, 14 Mar 2022 18:39:02 +0530
Subject: [PATCH] support parallel zstd compression
---
doc/src/sgml/ref/pg_basebackup.sgml | 11 ++-
src/backend/replication/basebackup.c | 14 +++-
src/backend/replication/basebackup_zstd.c | 15 ++++-
src/bin/pg_basebackup/bbstreamer.h | 3 +-
src/bin/pg_basebackup/bbstreamer_zstd.c | 11 ++-
src/bin/pg_basebackup/pg_basebackup.c | 97 +++++++++++++++++++++------
src/bin/pg_verifybackup/t/008_untar.pl | 8 +++
src/bin/pg_verifybackup/t/010_client_untar.pl | 8 +++
src/include/replication/basebackup_sink.h | 2 +-
9 files changed, 142 insertions(+), 27 deletions(-)
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 4a630b5..87feca0 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -399,9 +399,9 @@ PostgreSQL documentation
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
- <term><option>-Z [{client|server}-]<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>-Z [{client|server}-]<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>][@<replaceable>workers</replaceable>]</term>
<term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
- <term><option>--compress=[{client|server}-]<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=[{client|server}-]<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>][@<replaceable>workers</replaceable>]</term>
<listitem>
<para>
Requests compression of the backup. If <literal>client</literal> or
@@ -428,6 +428,13 @@ PostgreSQL documentation
the level is 0.
</para>
<para>
+ Compression workers can be specified optionally by appending the
+ number of workers after an at sign (<literal>@</literal>). It
+ defines the degree of parallelism while compressing the archive.
+ Currently, parallel compression is supported only for
+ <literal>zstd</literal> compressed archives.
+ </para>
+ <para>
When the tar format is used with <literal>gzip</literal>,
<literal>lz4</literal>, or <literal>zstd</literal>, the suffix
<filename>.gz</filename>, <filename>.lz4</filename>, or
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 2378ce5..8217fa9 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -82,6 +82,7 @@ typedef struct
backup_manifest_option manifest;
basebackup_compression_type compression;
int compression_level;
+ int compression_workers;
pg_checksum_type manifest_checksum_type;
} basebackup_options;
@@ -718,6 +719,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
char *target_str = "compat"; /* placate compiler */
bool o_compression = false;
bool o_compression_level = false;
+ bool o_compression_workers = false;
MemSet(opt, 0, sizeof(*opt));
opt->target = BACKUP_TARGET_CLIENT;
@@ -925,6 +927,15 @@ parse_basebackup_options(List *options, basebackup_options *opt)
opt->compression_level = defGetInt32(defel);
o_compression_level = true;
}
+ else if (strcmp(defel->defname, "compression_workers") == 0)
+ {
+ if (o_compression_workers)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("duplicate option \"%s\"", defel->defname)));
+ opt->compression_workers = defGetInt32(defel);
+ o_compression_workers = true;
+ }
else
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
@@ -1030,7 +1041,8 @@ SendBaseBackup(BaseBackupCmd *cmd)
else if (opt.compression == BACKUP_COMPRESSION_LZ4)
sink = bbsink_lz4_new(sink, opt.compression_level);
else if (opt.compression == BACKUP_COMPRESSION_ZSTD)
- sink = bbsink_zstd_new(sink, opt.compression_level);
+ sink = bbsink_zstd_new(sink, opt.compression_level,
+ opt.compression_workers);
/* Set up progress reporting. */
sink = bbsink_progress_new(sink, opt.progress);
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index e3f9b1d..54b91eb 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -28,6 +28,9 @@ typedef struct bbsink_zstd
/* Compression level */
int compresslevel;
+ /* Compression workers*/
+ int compressworkers;
+
ZSTD_CCtx *cctx;
ZSTD_outBuffer zstd_outBuf;
} bbsink_zstd;
@@ -59,7 +62,7 @@ const bbsink_ops bbsink_zstd_ops = {
* designated compression level.
*/
bbsink *
-bbsink_zstd_new(bbsink *next, int compresslevel)
+bbsink_zstd_new(bbsink *next, int compresslevel, int compressworkers)
{
#ifndef HAVE_LIBZSTD
ereport(ERROR,
@@ -81,6 +84,7 @@ bbsink_zstd_new(bbsink *next, int compresslevel)
*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
sink->base.bbs_next = next;
sink->compresslevel = compresslevel;
+ sink->compressworkers = compressworkers;
return &sink->base;
#endif
@@ -96,6 +100,7 @@ bbsink_zstd_begin_backup(bbsink *sink)
{
bbsink_zstd *mysink = (bbsink_zstd *) sink;
size_t output_buffer_bound;
+ size_t ret;
mysink->cctx = ZSTD_createCCtx();
if (!mysink->cctx)
@@ -104,6 +109,14 @@ bbsink_zstd_begin_backup(bbsink *sink)
ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
mysink->compresslevel);
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_nbWorkers,
+ mysink->compressworkers);
+
+ if (ZSTD_isError(ret))
+ elog(ERROR,
+ "could not compress data: %s",
+ ZSTD_getErrorName(ret));
+
/*
* We need our own buffer, because we're going to pass different data to
* the next sink than what gets passed to us.
diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h
index 02d4c05..dbaf6d6 100644
--- a/src/bin/pg_basebackup/bbstreamer.h
+++ b/src/bin/pg_basebackup/bbstreamer.h
@@ -210,7 +210,8 @@ extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,
int compresslevel);
extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next,
- int compresslevel);
+ int compresslevel,
+ int compressworkers);
extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index cc68367..f3d453e 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -63,10 +63,12 @@ const bbstreamer_ops bbstreamer_zstd_decompressor_ops = {
* blocks.
*/
bbstreamer *
-bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
+bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel,
+ int compressworkers)
{
#ifdef HAVE_LIBZSTD
bbstreamer_zstd_frame *streamer;
+ size_t ret;
Assert(next != NULL);
@@ -87,6 +89,13 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel)
ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
compresslevel);
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
+ compressworkers);
+
+ if (ZSTD_isError(ret))
+ pg_log_error("could not compress data: %s",
+ ZSTD_getErrorName(ret));
+
/* Initialize the ZSTD output buffer. */
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index d265ee3..55a321d 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -133,6 +133,7 @@ static bool showprogress = false;
static bool estimatesize = true;
static int verbose = 0;
static int compresslevel = 0;
+static int compressworkers = 0;
static WalCompressionMethod compressmethod = COMPRESSION_NONE;
static CompressionLocation compressloc = COMPRESS_LOCATION_UNSPECIFIED;
static IncludeWal includewal = STREAM_WAL;
@@ -405,8 +406,8 @@ usage(void)
printf(_(" -X, --wal-method=none|fetch|stream\n"
" include required WAL files with specified method\n"));
printf(_(" -z, --gzip compress tar output\n"));
- printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n"
- " compress tar output with given compression method or level\n"));
+ printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL][@WORKERS]\n"
+ " compress tar output with given compression method or level or workers\n"));
printf(_(" -Z, --compress=none do not compress tar output\n"));
printf(_("\nGeneral options:\n"));
printf(_(" -c, --checkpoint=fast|spread\n"
@@ -1005,29 +1006,36 @@ parse_max_rate(char *src)
/*
* Utility wrapper to parse the values specified for -Z/--compress.
- * *methodres and *levelres will be optionally filled with values coming
- * from the parsed results.
+ * *methodres, *levelres and *workerres will be optionally filled with values
+ * coming from the parsed result.
*/
static void
parse_compress_options(char *src, WalCompressionMethod *methodres,
- CompressionLocation *locationres, int *levelres)
+ CompressionLocation *locationres, int *levelres,
+ int *workerres)
{
char *sep;
- int firstlen;
- char *firstpart;
+ int firstlen,
+ secondlen;
+ char *firstpart,
+ *secondpart;
/*
- * clear 'levelres' so that if there are multiple compression options,
- * the last one fully overrides the earlier ones
+ * clear 'levelres' and 'workerres' so that if there are multiple
+ * compression options, the last one fully overrides the earlier ones.
*/
*levelres = 0;
+ *workerres = 0;
- /* check if the option is split in two */
+ /* check if the option is split in two using either ':' or '@'. */
sep = strchr(src, ':');
+ if (sep == NULL)
+ sep = strchr(src, '@');
+
/*
* The first part of the option value could be a method name, or just a
- * level value.
+ * level value or compression workers.
*/
firstlen = (sep != NULL) ? (sep - src) : strlen(src);
firstpart = pg_malloc(firstlen + 1);
@@ -1107,32 +1115,76 @@ parse_compress_options(char *src, WalCompressionMethod *methodres,
return;
}
+ /* Check for the second part of the input option. */
+ sep = strchr(src, ':');
+
+ if (sep != NULL)
+ {
+ /* Check the contents after the colon separator. */
+ sep++;
+ if (*sep == '\0')
+ {
+ pg_log_error("no compression level defined for method %s", firstpart);
+ exit(1);
+ }
+
+ /* Check if the option can be further split into two. */
+ src = sep;
+ sep = strchr(src, '@');
+
+ /* The second part of the value is compression level. */
+ secondlen = (sep != NULL) ? (sep - src) : strlen(src);
+ secondpart = pg_malloc(secondlen + 1);
+ memcpy(secondpart, src, secondlen);
+ secondpart[secondlen] = '\0';
+
+ /*
+ * For any of the methods currently supported, the data after the
+ * separator can just be an integer.
+ */
+ if (!option_parse_int(secondpart, "-Z/--compress", 0, INT_MAX,
+ levelres))
+ exit(1);
+
+ free(secondpart);
+ }
+
+ /* Check for the third part of the input option. */
+ sep = strchr(src, '@');
+
if (sep == NULL)
{
/*
- * The caller specified a method without a colon separator, so let any
- * subsequent checks assign a default level.
+ * The caller specified a method without a '@' separator, so let any
+ * subsequent checks assign a default number of workers.
*/
free(firstpart);
return;
}
- /* Check the contents after the colon separator. */
+ /* Check the contents after the '@' separator. */
sep++;
if (*sep == '\0')
{
- pg_log_error("no compression level defined for method %s", firstpart);
+ pg_log_error("compression workers are not defined for method %s", firstpart);
exit(1);
}
/*
- * For any of the methods currently supported, the data after the
- * separator can just be an integer.
+ * The data after '@' separator can just be an integer and it identifies
+ * the number of compression workers.
*/
if (!option_parse_int(sep, "-Z/--compress", 0, INT_MAX,
- levelres))
+ workerres))
exit(1);
+ if (*methodres != COMPRESSION_ZSTD)
+ {
+ pg_log_error("cannot use compression workers with method %s",
+ firstpart);
+ exit(1);
+ }
+
free(firstpart);
}
@@ -1341,7 +1393,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
streamer = bbstreamer_plain_writer_new(archive_filename,
archive_file);
streamer = bbstreamer_zstd_compressor_new(streamer,
- compresslevel);
+ compresslevel,
+ compressworkers);
}
else
{
@@ -2082,6 +2135,9 @@ BaseBackup(void)
if (compresslevel >= 1) /* not 0 or Z_DEFAULT_COMPRESSION */
AppendIntegerCommandOption(&buf, use_new_option_syntax,
"COMPRESSION_LEVEL", compresslevel);
+ if (compressworkers > 1)
+ AppendIntegerCommandOption(&buf, use_new_option_syntax,
+ "COMPRESSION_WORKERS", compressworkers);
}
if (verbose)
@@ -2626,7 +2682,8 @@ main(int argc, char **argv)
break;
case 'Z':
parse_compress_options(optarg, &compressmethod,
- &compressloc, &compresslevel);
+ &compressloc, &compresslevel,
+ &compressworkers);
break;
case 'c':
if (pg_strcasecmp(optarg, "fast") == 0)
diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl
index efbc910..86e2c8a 100644
--- a/src/bin/pg_verifybackup/t/008_untar.pl
+++ b/src/bin/pg_verifybackup/t/008_untar.pl
@@ -50,6 +50,14 @@ my @test_configuration = (
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'server-zstd@4'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index c2a6161..ac5ae31 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -50,6 +50,14 @@ my @test_configuration = (
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
+ },
+ {
+ 'compression_method' => 'zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:5@4'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define HAVE_LIBZSTD 1")
}
);
diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h
index a7f1675..5c1dd32 100644
--- a/src/include/replication/basebackup_sink.h
+++ b/src/include/replication/basebackup_sink.h
@@ -285,7 +285,7 @@ extern void bbsink_forward_cleanup(bbsink *sink);
extern bbsink *bbsink_copystream_new(bool send_to_client);
extern bbsink *bbsink_gzip_new(bbsink *next, int compresslevel);
extern bbsink *bbsink_lz4_new(bbsink *next, int compresslevel);
-extern bbsink *bbsink_zstd_new(bbsink *next, int compresslevel);
+extern bbsink *bbsink_zstd_new(bbsink *next, int compresslevel, int compressworkers);
extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size);
extern bbsink *bbsink_server_new(bbsink *next, char *pathname);
extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate);
--
1.8.3.1
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c (zstd workers)
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
@ 2022-03-14 16:35 ` Justin Pryzby <[email protected]>
2 siblings, 0 replies; 103+ messages in thread
From: Justin Pryzby @ 2022-03-14 16:35 UTC (permalink / raw)
To: Dipesh Pandit <[email protected]>; +Cc: Robert Haas <[email protected]>; Andres Freund <[email protected]>; Jeevan Ladhe <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Mon, Mar 14, 2022 at 09:41:35PM +0530, Dipesh Pandit wrote:
> I tried to implement support for parallel ZSTD compression. The
> library provides an option (ZSTD_c_nbWorkers) to specify the
> number of compression workers. The number of parallel
> workers can be set as part of compression parameter and if this
> option is specified then the library performs parallel compression
> based on the specified number of workers.
>
> User can specify the number of parallel worker as part of
> --compress option by appending an integer value after at sign (@).
> (-Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL][@WORKERS])
I suggest to use a syntax that's more general than that, maybe something like
:[level=]N,parallel=N,flag,flag,...
For example, someone may want to use zstd "long" mode or (when it's released)
rsyncable mode, or specify fine-grained compression parameters (strategy,
windowLog, hashLog, etc).
I hope the same syntax will be shared with wal_compression and pg_dump.
And libpq, if that patch progresses.
BTW, I think this may be better left for PG16.
--
Justin
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
@ 2022-03-15 10:33 ` Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2 siblings, 1 reply; 103+ messages in thread
From: Jeevan Ladhe @ 2022-03-15 10:33 UTC (permalink / raw)
To: Dipesh Pandit <[email protected]>; +Cc: Robert Haas <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
Thanks for the patch, Dipesh.
I had a look at the patch and also tried to take the backup. I have
following suggestions and observations:
I get following error at my end:
$ pg_basebackup -D /tmp/zstd_bk -Ft -Xfetch --compress=server-zstd:7@4
pg_basebackup: error: could not initiate base backup: ERROR: could not
compress data: Unsupported parameter
pg_basebackup: removing data directory "/tmp/zstd_bk"
This is mostly because I have the zstd library version v1.4.4, which
does not have default support for parallel workers. Maybe we should
have a better error, something that is hinting that the parallelism is
not supported by the particular build.
The regression for pg_verifybackup test 008_untar.pl also fails with a
similar error. Here, I think we should have some logic in regression to
skip the test if the parameter is not supported?
+ if (ZSTD_isError(ret))
+ elog(ERROR,
+ "could not compress data: %s",
+ ZSTD_getErrorName(ret));
I think all of this can go on one line, but anyhow we have to improve
the error message here.
Also, just a thought, for the versions where parallelism is not
supported, should we instead just throw a warning and fall back to
non-parallel behavior?
Regards,
Jeevan Ladhe
On Mon, 14 Mar 2022 at 21:41, Dipesh Pandit <[email protected]> wrote:
> Hi,
>
> I tried to implement support for parallel ZSTD compression. The
> library provides an option (ZSTD_c_nbWorkers) to specify the
> number of compression workers. The number of parallel
> workers can be set as part of compression parameter and if this
> option is specified then the library performs parallel compression
> based on the specified number of workers.
>
> User can specify the number of parallel worker as part of
> --compress option by appending an integer value after at sign (@).
> (-Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL][@WORKERS])
>
> Please find the attached patch v1 with the above changes.
>
> Note: ZSTD library version 1.5.x supports parallel compression
> by default and if the library version is lower than 1.5.x then
> parallel compression is enabled only the source is compiled with build
> macro ZSTD_MULTITHREAD. If the linked library version doesn't
> support parallel compression then setting the value of parameter
> ZSTD_c_nbWorkers to a value other than 0 will be no-op and
> returns an error.
>
> Thanks,
> Dipesh
>
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
@ 2022-03-15 17:50 ` Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-15 17:50 UTC (permalink / raw)
To: Jeevan Ladhe <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Shinoda, Noriyoshi (PN Japan FSIP) <[email protected]>; Abhijit Menon-Sen <[email protected]>; Dmitry Dolgov <[email protected]>; Jeevan Ladhe <[email protected]>; Mark Dilger <[email protected]>; pgsql-hackers; tushar <[email protected]>
On Tue, Mar 15, 2022 at 6:33 AM Jeevan Ladhe <[email protected]> wrote:
> I get following error at my end:
>
> $ pg_basebackup -D /tmp/zstd_bk -Ft -Xfetch --compress=server-zstd:7@4
> pg_basebackup: error: could not initiate base backup: ERROR: could not compress data: Unsupported parameter
> pg_basebackup: removing data directory "/tmp/zstd_bk"
>
> This is mostly because I have the zstd library version v1.4.4, which
> does not have default support for parallel workers. Maybe we should
> have a better error, something that is hinting that the parallelism is
> not supported by the particular build.
I'm not averse to trying to improve that error message, but honestly
I'd consider that to be good enough already to be acceptable. We could
think about trying to add an errhint() telling you that the problem
may be with your libzstd build.
> The regression for pg_verifybackup test 008_untar.pl also fails with a
> similar error. Here, I think we should have some logic in regression to
> skip the test if the parameter is not supported?
Or at least to have the test not fail.
> Also, just a thought, for the versions where parallelism is not
> supported, should we instead just throw a warning and fall back to
> non-parallel behavior?
I don't think so. I think it's better for the user to get an error and
then change their mind and request something we can do.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2023-03-23 02:08 ` Thomas Munro <[email protected]>
2023-03-23 20:11 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Thomas Munro @ 2023-03-23 02:08 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: pgsql-hackers
On Thu, Mar 23, 2023 at 2:50 PM Thomas Munro <[email protected]> wrote:
> In rem: commit 3500ccc3,
>
> for X in ` grep -E '^[^*]+event_name = "'
> src/backend/utils/activity/wait_event.c |
> sed 's/^.* = "//;s/";$//;/unknown/d' `
> do
> if ! git grep "$X" doc/src/sgml/monitoring.sgml > /dev/null
> then
> echo "$X is not documented"
> fi
> done
>
> BaseBackupSync is not documented
> BaseBackupWrite is not documented
[Resending with trimmed CC: list, because the mailing list told me to
due to a blocked account, sorry if you already got the above.]
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
@ 2023-03-23 20:11 ` Robert Haas <[email protected]>
2023-03-24 14:46 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2023-03-23 20:11 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: pgsql-hackers
On Wed, Mar 22, 2023 at 10:09 PM Thomas Munro <[email protected]> wrote:
> > BaseBackupSync is not documented
> > BaseBackupWrite is not documented
>
> [Resending with trimmed CC: list, because the mailing list told me to
> due to a blocked account, sorry if you already got the above.]
Bummer. I'll write a patch to fix that tomorrow, unless somebody beats me to it.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
2023-03-23 20:11 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2023-03-24 14:46 ` Robert Haas <[email protected]>
2023-04-12 14:57 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2023-03-24 14:46 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: pgsql-hackers; Peter Eisentraut <[email protected]>
On Thu, Mar 23, 2023 at 4:11 PM Robert Haas <[email protected]> wrote:
> On Wed, Mar 22, 2023 at 10:09 PM Thomas Munro <[email protected]> wrote:
> > > BaseBackupSync is not documented
> > > BaseBackupWrite is not documented
> >
> > [Resending with trimmed CC: list, because the mailing list told me to
> > due to a blocked account, sorry if you already got the above.]
>
> Bummer. I'll write a patch to fix that tomorrow, unless somebody beats me to it.
Here's a patch for that, and a patch to add the missing error check
Peter noticed.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] 0001-Add-missing-documentation-entries-for-new-base-backu.patch (1.2K, ../../CA+TgmoaDMmt=dvOMZE_D9EtViih7O7FYGm9NVjH=HCrNU7KYOQ@mail.gmail.com/2-0001-Add-missing-documentation-entries-for-new-base-backu.patch)
download | inline diff:
From c2c6395c2c38eaedeac6a9c045bd4a8b791eb414 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Fri, 24 Mar 2023 10:37:33 -0400
Subject: [PATCH 1/2] Add missing documentation entries for new base backup
wait events.
Per complaint from Thomas Munro.
---
doc/src/sgml/monitoring.sgml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 21e6ce2841..488b76c765 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1272,6 +1272,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<entry><literal>BaseBackupRead</literal></entry>
<entry>Waiting for base backup to read from a file.</entry>
</row>
+ <row>
+ <entry><literal>BaseBackupSync</literal></entry>
+ <entry>Waiting for data written by a base backup to reach durable storage.</entry>
+ </row>
+ <row>
+ <entry><literal>BaseBackupWrite</literal></entry>
+ <entry>Waiting for base backup to write to a file.</entry>
+ </row>
<row>
<entry><literal>BufFileRead</literal></entry>
<entry>Waiting for a read from a buffered file.</entry>
--
2.37.1 (Apple Git-137.1)
[application/octet-stream] 0002-basebackup_to_shell-Add-missing-error-check.patch (985B, ../../CA+TgmoaDMmt=dvOMZE_D9EtViih7O7FYGm9NVjH=HCrNU7KYOQ@mail.gmail.com/3-0002-basebackup_to_shell-Add-missing-error-check.patch)
download | inline diff:
From 509985230a3da4b9c6047ad38094f061660c059a Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Fri, 24 Mar 2023 10:44:03 -0400
Subject: [PATCH 2/2] basebackup_to_shell: Add missing error check.
Per complaint from Peter Eisentraut.
---
contrib/basebackup_to_shell/basebackup_to_shell.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/contrib/basebackup_to_shell/basebackup_to_shell.c b/contrib/basebackup_to_shell/basebackup_to_shell.c
index 29f5069d42..57ed587d48 100644
--- a/contrib/basebackup_to_shell/basebackup_to_shell.c
+++ b/contrib/basebackup_to_shell/basebackup_to_shell.c
@@ -263,6 +263,11 @@ shell_run_command(bbsink_shell *sink, const char *filename)
/* Run it. */
sink->pipe = OpenPipeStream(sink->current_command, PG_BINARY_W);
+ if (sink->pipe == NULL)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not execute command \"%s\": %m",
+ sink->current_command)));
}
/*
--
2.37.1 (Apple Git-137.1)
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
2023-03-23 20:11 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-24 14:46 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
@ 2023-04-12 14:57 ` Justin Pryzby <[email protected]>
2023-04-12 15:55 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2023-04-12 14:57 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Thomas Munro <[email protected]>; pgsql-hackers; Peter Eisentraut <[email protected]>
On Fri, Mar 24, 2023 at 10:46:37AM -0400, Robert Haas wrote:
> On Thu, Mar 23, 2023 at 4:11 PM Robert Haas <[email protected]> wrote:
> > On Wed, Mar 22, 2023 at 10:09 PM Thomas Munro <[email protected]> wrote:
> > > > BaseBackupSync is not documented
> > > > BaseBackupWrite is not documented
> > >
> > > [Resending with trimmed CC: list, because the mailing list told me to
> > > due to a blocked account, sorry if you already got the above.]
> >
> > Bummer. I'll write a patch to fix that tomorrow, unless somebody beats me to it.
>
> Here's a patch for that, and a patch to add the missing error check
> Peter noticed.
I think these maybe got forgotten ?
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: refactoring basebackup.c
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
2023-03-23 20:11 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-24 14:46 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-04-12 14:57 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
@ 2023-04-12 15:55 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Robert Haas @ 2023-04-12 15:55 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Thomas Munro <[email protected]>; pgsql-hackers; Peter Eisentraut <[email protected]>
On Wed, Apr 12, 2023 at 10:57 AM Justin Pryzby <[email protected]> wrote:
> I think these maybe got forgotten ?
Committed.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
@ 2022-03-23 20:34 ` Robert Haas <[email protected]>
2022-03-23 21:14 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 23:07 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 12:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2 siblings, 5 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-23 20:34 UTC (permalink / raw)
To: Dipesh Pandit <[email protected]>; +Cc: Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
[ Changing subject line in the hopes of attracting more eyeballs. ]
On Mon, Mar 14, 2022 at 12:11 PM Dipesh Pandit <[email protected]> wrote:
> I tried to implement support for parallel ZSTD compression.
Here's a new patch for this. It's more of a rewrite than an update,
honestly; commit ffd53659c46a54a6978bcb8c4424c1e157a2c0f1 necessitated
totally different options handling, but I also redid the test cases,
the documentation, and the error message.
For those who may not have been following along, here's an executive
summary: libzstd offers an option for parallel compression. It's
intended to be transparent: you just say you want it, and the library
takes care of it for you. Since we have the ability to do backup
compression on either the client or the server side, we can expose
this option in both locations. That would be cool, because it would
allow for really fast backup compression with a good compression
ratio. It would also mean that we would be, or really libzstd would
be, spawning threads inside the PostgreSQL backend. Short of cats and
dogs living together, it's hard to think of anything more terrifying,
because the PostgreSQL backend is very much not thread-safe. However,
a lot of the things we usually worry about when people make noises
about using threads in the backend don't apply here, because the
threads are hidden away behind libzstd interfaces and can't execute
any PostgreSQL code. Therefore, I think it might be safe to just ...
turn this on. One reason I think that is that this whole approach was
recommended to me by Andres ... but that's not to say that there
couldn't be problems. I worry a bit that the mere presence of threads
could in some way mess things up, but I don't know what the mechanism
for that would be, and I don't want to postpone shipping useful
features based on nebulous fears.
In my ideal world, I'd like to push this into v15. I've done a lot of
work to improve the backup code in this release, and this is actually
a very small change yet one that potentially enables the project to
get a lot more value out of the work that has already been committed.
That said, I also don't want to break the world, so if you have an
idea what this would break, please tell me.
For those curious as to how this affects performance and backup size,
I loaded up the UK land registry database. That creates a 3769MB
database. Then I backed it up using client-side compression and
server-side compression using the various different algorithms that
are supported in the master branch, plus parallel zstd.
no compression: 3.7GB, 9 seconds
gzip: 1.5GB, 140 seconds with server-side, 141 seconds with client-side
lz4: 2.0GB, 13 seconds with server-side, 12 seconds with client-side
For both parallel and non-parallel zstd compression, I see differences
between the compressed size depending on where the compression is
done. I don't know whether this is an expected behavior of the zstd
library or a bug. Both files uncompress OK and pass pg_verifybackup,
but that doesn't mean we're not, for example, selecting different
compression levels where we shouldn't be. I'll try to figure out
what's going on here.
zstd, client-side: 1.7GB, 17 seconds
zstd, server-side: 1.3GB, 25 seconds
parallel zstd, 4 workers, client-side: 1.7GB, 7.5 seconds
parallel zstd, 4 workers, server-side: 1.3GB, 7.2 seconds
Notice that compressing the backup with parallel zstd is actually
faster than taking an uncompressed backup, even though this test is
all being run on the same machine. That's kind of crazy to me: the
parallel compression is so fast that we save more time on I/O than we
spend compressing. This assumes of course that you have plenty of CPU
resources and limited I/O resources, which won't be true for everyone,
but it's not an unusual situation.
I think the documentation changes in this patch might not be quite up
to scratch. I think there's a brewing problem here: as we add more
compression options, whether or not that happens in this release, and
regardless of what specific options we add, the way things are
structured right now, we're going to end up either duplicating a bunch
of stuff between the pg_basebackup documentation and the BASE_BACKUP
documentation, or else one of those places is going to end up lacking
information that someone reading it might like to have. I'm not
exactly sure what to do about this, though.
This patch contains a trivial adjustment to
PostgreSQL::Test::Cluster::run_log to make it return a useful value
instead of not. I think that should be pulled out and committed
independently regardless of what happens to this patch overall, and
possibly back-patched.
Thanks,
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] 0001-Allow-parallel-zstd-compression-when-taking-a-base-b.patch (12.9K, ../../CA+Tgmobj6u-nWF-j=FemygUhobhryLxf9h-wJN7W-2rSsseHNA@mail.gmail.com/2-0001-Allow-parallel-zstd-compression-when-taking-a-base-b.patch)
download | inline diff:
From bf27b972eaf29c0a40b949eac40150a1d9ee00b0 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Wed, 23 Mar 2022 11:00:33 -0400
Subject: [PATCH] Allow parallel zstd compression when taking a base backup.
libzstd allows transparent parallel compression just by setting
an option when creating the compression context, so permit that
for both client and server-side backup compression. To use this,
use something like pg_basebackup --compress WHERE-zstd:workers=N
where WHERE is "client" or "server" and N is an integer.
When compression is performed on the server side, this will spawn
threads inside the PostgreSQL backend. While there is almost no
PostgreSQL server code which is thread-safe, the threads here are used
internally by libzstd and touch only data structures controlled by
libzstd.
Patch by me, based in part on earlier work by Dipesh Pandit
and Jeevan Ladhe.
---
doc/src/sgml/protocol.sgml | 12 +++++--
doc/src/sgml/ref/pg_basebackup.sgml | 4 +--
src/backend/replication/basebackup_zstd.c | 19 +++++++++++
src/bin/pg_basebackup/bbstreamer_zstd.c | 16 +++++++++
src/bin/pg_basebackup/t/010_pg_basebackup.pl | 5 +++
src/bin/pg_verifybackup/t/009_extract.pl | 29 ++++++++++++++--
src/bin/pg_verifybackup/t/010_client_untar.pl | 33 +++++++++++++++++--
src/common/backup_compression.c | 16 +++++++++
src/include/common/backup_compression.h | 2 ++
src/test/perl/PostgreSQL/Test/Cluster.pm | 3 +-
10 files changed, 127 insertions(+), 12 deletions(-)
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 719b947ef4..cc03a4587b 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2739,17 +2739,23 @@ The commands accepted in replication mode are:
option. If the value is an integer, it specifies the compression
level. Otherwise, it should be a comma-separated list of items,
each of the form <literal>keyword</literal> or
- <literal>keyword=value</literal>. Currently, the only supported
- keyword is <literal>level</literal>, which sets the compression
- level.
+ <literal>keyword=value</literal>. Currently, the supported keywords
+ are <literal>level</literal> and <literal>workers</literal>.
</para>
<para>
+ The <literal>level</literal> keyword sets the compression level.
For <literal>gzip</literal> the compression level should be an
integer between 1 and 9, for <literal>lz4</literal> an integer
between 1 and 12, and for <literal>zstd</literal> an integer
between 1 and 22.
</para>
+
+ <para>
+ The <literal>workers</literal> keyword sets the number of threads
+ that should be used for parallel compression. Parallel compression
+ is supported only for <literal>zstd</literal>.
+ </para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index d9233beb8e..82f5f60625 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -424,8 +424,8 @@ PostgreSQL documentation
integer, it specifies the compression level. Otherwise, it should be
a comma-separated list of items, each of the form
<literal>keyword</literal> or <literal>keyword=value</literal>.
- Currently, the only supported keyword is <literal>level</literal>,
- which sets the compression level.
+ Currently, the supported keywords are <literal>level</literal>
+ and <literal>workers</literal>.
</para>
<para>
If no compression level is specified, the default compression level
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index bb5b668c2a..4835aa70fc 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -28,6 +28,9 @@ typedef struct bbsink_zstd
/* Compression level */
int compresslevel;
+ /* Number of parallel workers. */
+ int workers;
+
ZSTD_CCtx *cctx;
ZSTD_outBuffer zstd_outBuf;
} bbsink_zstd;
@@ -83,6 +86,7 @@ bbsink_zstd_new(bbsink *next, bc_specification *compress)
*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
sink->base.bbs_next = next;
sink->compresslevel = compresslevel;
+ sink->workers = compress->workers;
return &sink->base;
#endif
@@ -98,6 +102,7 @@ bbsink_zstd_begin_backup(bbsink *sink)
{
bbsink_zstd *mysink = (bbsink_zstd *) sink;
size_t output_buffer_bound;
+ size_t ret;
mysink->cctx = ZSTD_createCCtx();
if (!mysink->cctx)
@@ -106,6 +111,20 @@ bbsink_zstd_begin_backup(bbsink *sink)
ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
mysink->compresslevel);
+ /*
+ * We check for failure here because (1) older versions of the library
+ * do not support ZSTD_c_nbWorkers and (2) the library might want to
+ * reject an unreasonable values (though in practice it does not seem to do
+ * so).
+ */
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_nbWorkers,
+ mysink->workers);
+ if (ZSTD_isError(ret))
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not set compression worker count to %d: %s",
+ mysink->workers, ZSTD_getErrorName(ret)));
+
/*
* We need our own buffer, because we're going to pass different data to
* the next sink than what gets passed to us.
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index caa5edcaf1..e17dfb6bd5 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -67,6 +67,7 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
{
#ifdef USE_ZSTD
bbstreamer_zstd_frame *streamer;
+ size_t ret;
Assert(next != NULL);
@@ -87,6 +88,21 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
compress->level);
+ /*
+ * We check for failure here because (1) older versions of the library
+ * do not support ZSTD_c_nbWorkers and (2) the library might want to
+ * reject unreasonable values (though in practice it does not seem to do
+ * so).
+ */
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
+ compress->workers);
+ if (ZSTD_isError(ret))
+ {
+ pg_log_error("could not set compression worker count to %d: %s",
+ compress->workers, ZSTD_getErrorName(ret));
+ exit(1);
+ }
+
/* Initialize the ZSTD output buffer. */
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 2869a239e7..f074fe19b7 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -133,6 +133,11 @@ my @compression_failure_tests = (
'invalid compression specification: found empty string where a compression option was expected',
'failure on extra, empty compression option'
],
+ [
+ 'gzip:workers=3',
+ 'invalid compression specification: compression algorithm "gzip" does not accept a worker count',
+ 'failure on worker count for gzip'
+ ],
);
for my $cft (@compression_failure_tests)
{
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index 9f9cc7540b..e17e7cad51 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -36,6 +36,12 @@ my @test_configuration = (
'compression_method' => 'zstd',
'backup_flags' => ['--compress', 'server-zstd:5'],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'server-zstd:workers=3'],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -57,8 +63,27 @@ for my $tc (@test_configuration)
my @verify = ('pg_verifybackup', '-e', $backup_path);
# A backup with a valid compression method should work.
- $primary->command_ok(\@backup,
- "backup done, compression method \"$method\"");
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 2;
+ }
+ else
+ {
+ ok($backup_result, "backup done, compression $method");
+ }
# Make sure that it verifies OK.
$primary->command_ok(\@verify,
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 487e30e826..5f6a4b9963 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -50,6 +50,15 @@ my @test_configuration = (
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:workers=3'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -70,9 +79,27 @@ for my $tc (@test_configuration)
'pg_basebackup', '-D', $backup_path,
'-Xfetch', '--no-sync', '-cfast', '-Ft');
push @backup, @{$tc->{'backup_flags'}};
- $primary->command_ok(\@backup,
- "client side backup, compression $method");
-
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 3;
+ }
+ else
+ {
+ ok($backup_result, "client side backup, compression $method");
+ }
# Verify that the we got the files we expected.
my $backup_files = join(',',
diff --git a/src/common/backup_compression.c b/src/common/backup_compression.c
index 0650f975c4..969e08cca2 100644
--- a/src/common/backup_compression.c
+++ b/src/common/backup_compression.c
@@ -177,6 +177,11 @@ parse_bc_specification(bc_algorithm algorithm, char *specification,
result->level = expect_integer_value(keyword, value, result);
result->options |= BACKUP_COMPRESSION_OPTION_LEVEL;
}
+ else if (strcmp(keyword, "workers") == 0)
+ {
+ result->workers = expect_integer_value(keyword, value, result);
+ result->options |= BACKUP_COMPRESSION_OPTION_WORKERS;
+ }
else
result->parse_error =
psprintf(_("unknown compression option \"%s\""), keyword);
@@ -266,5 +271,16 @@ validate_bc_specification(bc_specification *spec)
min_level, max_level);
}
+ /*
+ * Of the compression algorithms that we currently support, only zstd
+ * allows parallel workers.
+ */
+ if ((spec->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0 &&
+ (spec->algorithm != BACKUP_COMPRESSION_ZSTD))
+ {
+ return psprintf(_("compression algorithm \"%s\" does not accept a worker count"),
+ get_bc_algorithm_name(spec->algorithm));
+ }
+
return NULL;
}
diff --git a/src/include/common/backup_compression.h b/src/include/common/backup_compression.h
index 0565cbc657..6a0ecaa99c 100644
--- a/src/include/common/backup_compression.h
+++ b/src/include/common/backup_compression.h
@@ -23,12 +23,14 @@ typedef enum bc_algorithm
} bc_algorithm;
#define BACKUP_COMPRESSION_OPTION_LEVEL (1 << 0)
+#define BACKUP_COMPRESSION_OPTION_WORKERS (1 << 1)
typedef struct bc_specification
{
bc_algorithm algorithm;
unsigned options; /* OR of BACKUP_COMPRESSION_OPTION constants */
int level;
+ int workers;
char *parse_error; /* NULL if parsing was OK, else message */
} bc_specification;
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index e7b9161137..8d838f7d6d 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2503,8 +2503,7 @@ sub run_log
local %ENV = $self->_get_env();
- PostgreSQL::Test::Utils::run_log(@_);
- return;
+ return PostgreSQL::Test::Utils::run_log(@_);
}
=pod
--
2.24.3 (Apple Git-128)
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-23 21:14 ` Andres Freund <[email protected]>
2022-03-23 22:31 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
4 siblings, 1 reply; 103+ messages in thread
From: Andres Freund @ 2022-03-23 21:14 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Hi,
On 2022-03-23 16:34:04 -0400, Robert Haas wrote:
> Therefore, I think it might be safe to just ... turn this on. One reason I
> think that is that this whole approach was recommended to me by Andres ...
I didn't do a super careful analysis of the issues... But I do think it's
pretty much the one case where it "should" be safe.
The most likely source of problem would errors thrown while zstd threads are
alive. Should make sure that that can't happen.
What is the lifetime of the threads zstd spawns? Are they tied to a single
compression call? A single ZSTD_createCCtx()? If the latter, how bulletproof
is our code ensuring that we don't leak such contexts?
If they're short-lived, are we compressing large enough batches to not waste a
lot of time starting/stopping threads?
> but that's not to say that there couldn't be problems. I worry a bit that
> the mere presence of threads could in some way mess things up, but I don't
> know what the mechanism for that would be, and I don't want to postpone
> shipping useful features based on nebulous fears.
One thing that'd be good to tests for is cancelling in-progress server-side
compression. And perhaps a few assertions that ensure that we don't escape
with some threads still running. That'd have to be platform dependent, but I
don't see a problem with that in this case.
> For both parallel and non-parallel zstd compression, I see differences
> between the compressed size depending on where the compression is
> done. I don't know whether this is an expected behavior of the zstd
> library or a bug. Both files uncompress OK and pass pg_verifybackup,
> but that doesn't mean we're not, for example, selecting different
> compression levels where we shouldn't be. I'll try to figure out
> what's going on here.
>
> zstd, client-side: 1.7GB, 17 seconds
> zstd, server-side: 1.3GB, 25 seconds
> parallel zstd, 4 workers, client-side: 1.7GB, 7.5 seconds
> parallel zstd, 4 workers, server-side: 1.3GB, 7.2 seconds
What causes this fairly massive client-side/server-side size difference?
> + /*
> + * We check for failure here because (1) older versions of the library
> + * do not support ZSTD_c_nbWorkers and (2) the library might want to
> + * reject unreasonable values (though in practice it does not seem to do
> + * so).
> + */
> + ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
> + compress->workers);
> + if (ZSTD_isError(ret))
> + {
> + pg_log_error("could not set compression worker count to %d: %s",
> + compress->workers, ZSTD_getErrorName(ret));
> + exit(1);
> + }
Will this cause test failures on systems with older zstd?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:14 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
@ 2022-03-23 22:31 ` Robert Haas <[email protected]>
2022-03-23 23:31 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-23 22:31 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 5:14 PM Andres Freund <[email protected]> wrote:
> The most likely source of problem would errors thrown while zstd threads are
> alive. Should make sure that that can't happen.
>
> What is the lifetime of the threads zstd spawns? Are they tied to a single
> compression call? A single ZSTD_createCCtx()? If the latter, how bulletproof
> is our code ensuring that we don't leak such contexts?
I haven't found any real documentation explaining how libzstd manages
its threads. I am assuming that it is tied to the ZSTD_CCtx, but I
don't know. I guess I could try to figure it out from the source code.
Anyway, what we have now is a PG_TRY()/PG_CATCH() block around the
code that uses the basink which will cause bbsink_zstd_cleanup() to
get called in the event of an error. That will do ZSTD_freeCCtx().
It's probably also worth mentioning here that even if, contrary to
expectations, the compression threads hang around to the end of time
and chill, in practice nobody is likely to run BASE_BACKUP and then
keep the connection open for a long time afterward. So it probably
wouldn't really affect resource utilization in real-world scenarios
even if the threads never exited, as long as they didn't, you know,
busy-loop in the background. And I assume the actual library behavior
can't be nearly that bad. This is a pretty mainstream piece of
software.
> If they're short-lived, are we compressing large enough batches to not waste a
> lot of time starting/stopping threads?
Well, we're using a single ZSTD_CCtx for an entire base backup. Again,
I haven't found documentation explaining with libzstd is actually
doing, but it's hard to see how we could make the batch any bigger
than that. The context gets reset for each new tablespace, which may
or may not do anything to the compression threads.
> > but that's not to say that there couldn't be problems. I worry a bit that
> > the mere presence of threads could in some way mess things up, but I don't
> > know what the mechanism for that would be, and I don't want to postpone
> > shipping useful features based on nebulous fears.
>
> One thing that'd be good to tests for is cancelling in-progress server-side
> compression. And perhaps a few assertions that ensure that we don't escape
> with some threads still running. That'd have to be platform dependent, but I
> don't see a problem with that in this case.
More specific suggestions, please?
> > For both parallel and non-parallel zstd compression, I see differences
> > between the compressed size depending on where the compression is
> > done. I don't know whether this is an expected behavior of the zstd
> > library or a bug. Both files uncompress OK and pass pg_verifybackup,
> > but that doesn't mean we're not, for example, selecting different
> > compression levels where we shouldn't be. I'll try to figure out
> > what's going on here.
> >
> > zstd, client-side: 1.7GB, 17 seconds
> > zstd, server-side: 1.3GB, 25 seconds
> > parallel zstd, 4 workers, client-side: 1.7GB, 7.5 seconds
> > parallel zstd, 4 workers, server-side: 1.3GB, 7.2 seconds
>
> What causes this fairly massive client-side/server-side size difference?
You seem not to have read what I wrote about this exact point in the
text which you quoted.
> Will this cause test failures on systems with older zstd?
I put a bunch of logic in the test case to try to avoid that, so
hopefully not, but if it does, we can adjust the logic.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:14 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-23 22:31 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-23 23:31 ` Andres Freund <[email protected]>
2022-03-24 13:39 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Andres Freund @ 2022-03-23 23:31 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Hi,
On 2022-03-23 18:31:12 -0400, Robert Haas wrote:
> On Wed, Mar 23, 2022 at 5:14 PM Andres Freund <[email protected]> wrote:
> > The most likely source of problem would errors thrown while zstd threads are
> > alive. Should make sure that that can't happen.
> >
> > What is the lifetime of the threads zstd spawns? Are they tied to a single
> > compression call? A single ZSTD_createCCtx()? If the latter, how bulletproof
> > is our code ensuring that we don't leak such contexts?
>
> I haven't found any real documentation explaining how libzstd manages
> its threads. I am assuming that it is tied to the ZSTD_CCtx, but I
> don't know. I guess I could try to figure it out from the source code.
I found this the following section in the manual [1]:
ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
* When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
* while compression is performed in parallel, within worker thread(s).
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
* compression is performed inside Caller's thread, and all invocations are blocking */
"ZSTD_compressStream*() consumes input ... immediately gives back control"
pretty much confirms that.
Do we care about zstd's memory usage here? I think it's OK to mostly ignore
work_mem/maintenance_work_mem here, but I could also see limiting concurrency
so that estimated memory usage would fit into work_mem/maintenance_work_mem.
> It's probably also worth mentioning here that even if, contrary to
> expectations, the compression threads hang around to the end of time
> and chill, in practice nobody is likely to run BASE_BACKUP and then
> keep the connection open for a long time afterward. So it probably
> wouldn't really affect resource utilization in real-world scenarios
> even if the threads never exited, as long as they didn't, you know,
> busy-loop in the background. And I assume the actual library behavior
> can't be nearly that bad. This is a pretty mainstream piece of
> software.
I'm not really worried about resource utilization, more about the existence of
threads moving us into undefined behaviour territory or such. I don't think
that's possible, but it's IIRC UB to fork() while threads are present and do
pretty much *anything* other than immediately exec*().
> > > but that's not to say that there couldn't be problems. I worry a bit that
> > > the mere presence of threads could in some way mess things up, but I don't
> > > know what the mechanism for that would be, and I don't want to postpone
> > > shipping useful features based on nebulous fears.
> >
> > One thing that'd be good to tests for is cancelling in-progress server-side
> > compression. And perhaps a few assertions that ensure that we don't escape
> > with some threads still running. That'd have to be platform dependent, but I
> > don't see a problem with that in this case.
>
> More specific suggestions, please?
I was thinking of doing something like calling pthread_is_threaded_np() before
and after the zstd section and erroring out if they differ. But I forgot that
that's on mac-ism.
> > > For both parallel and non-parallel zstd compression, I see differences
> > > between the compressed size depending on where the compression is
> > > done. I don't know whether this is an expected behavior of the zstd
> > > library or a bug. Both files uncompress OK and pass pg_verifybackup,
> > > but that doesn't mean we're not, for example, selecting different
> > > compression levels where we shouldn't be. I'll try to figure out
> > > what's going on here.
> > >
> > > zstd, client-side: 1.7GB, 17 seconds
> > > zstd, server-side: 1.3GB, 25 seconds
> > > parallel zstd, 4 workers, client-side: 1.7GB, 7.5 seconds
> > > parallel zstd, 4 workers, server-side: 1.3GB, 7.2 seconds
> >
> > What causes this fairly massive client-side/server-side size difference?
>
> You seem not to have read what I wrote about this exact point in the
> text which you quoted.
Somehow not...
Perhaps it's related to the amounts of memory fed to ZSTD_compressStream2() in
one invocation? I recall that there's some differences between basebackup
client / serverside around buffer sizes - but that's before all the recent-ish
changes...
Greetings,
Andres Freund
[1] http://facebook.github.io/zstd/zstd_manual.html
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:14 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-23 22:31 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 23:31 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
@ 2022-03-24 13:39 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-24 13:39 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 7:31 PM Andres Freund <[email protected]> wrote:
> I found this the following section in the manual [1]:
>
> ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
> * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
> * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
> * while compression is performed in parallel, within worker thread(s).
> * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
> * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
> * More workers improve speed, but also increase memory usage.
> * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
> * compression is performed inside Caller's thread, and all invocations are blocking */
>
> "ZSTD_compressStream*() consumes input ... immediately gives back control"
> pretty much confirms that.
I saw that too, but I didn't consider it conclusive. It would be nice
if their documentation had a bit more detail on what's really
happening.
> Do we care about zstd's memory usage here? I think it's OK to mostly ignore
> work_mem/maintenance_work_mem here, but I could also see limiting concurrency
> so that estimated memory usage would fit into work_mem/maintenance_work_mem.
I think it's possible that we want to do nothing and possible that we
want to do something, but I think it's very unlikely that the thing we
want to do is related to maintenance_work_mem. Say we soft-cap the
compression level to the one which we think will fit within
maintanence_work_mem. I think the most likely outcome is that people
will not get the compression level they request and be confused about
why that has happened. It also seems possible that we'll be wrong
about how much memory will be used - say, because somebody changes the
library behavior in a new release - and will limit it to the wrong
level. If we're going to do anything here, I think it should be to
limit based on the compression level itself and not based how much
memory we think that level will use.
But that leaves the question of whether we should even try to impose
some kind of limit, and there I'm not sure. It feels like it might be
overengineered, because we're only talking about users who have
replication privileges, and if those accounts are subverted there are
big problems anyway. I think if we imposed a governance system here it
would get very little use. On the other hand, I think that the higher
zstd compression levels of 20+ can actually use a ton of memory, so we
might want to limit access to those somehow. Apparently on the command
line you have to say --ultra -- not sure if there's a corresponding
API call or if that's a guard that's built specifically into the CLI.
> Perhaps it's related to the amounts of memory fed to ZSTD_compressStream2() in
> one invocation? I recall that there's some differences between basebackup
> client / serverside around buffer sizes - but that's before all the recent-ish
> changes...
That thought occurred to me too but I haven't investigated yet.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-23 21:52 ` Justin Pryzby <[email protected]>
2022-03-23 22:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 21:56 ` fixing a few backup compression goofs Robert Haas <[email protected]>
4 siblings, 2 replies; 103+ messages in thread
From: Justin Pryzby @ 2022-03-23 21:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
+ * We check for failure here because (1) older versions of the library
+ * do not support ZSTD_c_nbWorkers and (2) the library might want to
+ * reject an unreasonable values (though in practice it does not seem to do
+ * so).
+ */
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_nbWorkers,
+ mysink->workers);
+ if (ZSTD_isError(ret))
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not set compression worker count to %d: %s",
+ mysink->workers, ZSTD_getErrorName(ret)));
Also because the library may not be compiled with threading. A few days ago, I
tried to rebase the original "parallel workers" patch over the COMPRESS DETAIL
patch but then couldn't test it, even after trying various versions of the zstd
package and trying to compile it locally. I'll try again soon...
I think you should also test the return value when setting the compress level.
Not only because it's generally a good idea, but also because I suggested to
support negative compression levels. Which weren't allowed before v1.3.4, and
then the range is only defined since 1.3.6 (ZSTD_minCLevel). At some point,
the range may have been -7..22 but now it's -131072..22.
lib/compress/zstd_compress.c:int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
lib/zstd.h:#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
lib/zstd.h:#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
lib/zstd.h:#define ZSTD_BLOCKSIZELOG_MAX 17
; -1<<17
-131072
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
@ 2022-03-23 22:57 ` Robert Haas <[email protected]>
2022-03-27 20:50 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-23 22:57 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 5:52 PM Justin Pryzby <[email protected]> wrote:
> Also because the library may not be compiled with threading. A few days ago, I
> tried to rebase the original "parallel workers" patch over the COMPRESS DETAIL
> patch but then couldn't test it, even after trying various versions of the zstd
> package and trying to compile it locally. I'll try again soon...
Ah. Right, I can update the comment to mention that.
> I think you should also test the return value when setting the compress level.
> Not only because it's generally a good idea, but also because I suggested to
> support negative compression levels. Which weren't allowed before v1.3.4, and
> then the range is only defined since 1.3.6 (ZSTD_minCLevel). At some point,
> the range may have been -7..22 but now it's -131072..22.
Yeah, I was thinking that might be a good change. It would require
adjusting some other code though, because right now only compression
levels 1..22 are accepted anyhow.
> lib/compress/zstd_compress.c:int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
> lib/zstd.h:#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
> lib/zstd.h:#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
> lib/zstd.h:#define ZSTD_BLOCKSIZELOG_MAX 17
> ; -1<<17
> -131072
So does that, like, compress the value by making it way bigger? :-)
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 22:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-27 20:50 ` Justin Pryzby <[email protected]>
2022-03-28 16:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2022-03-27 20:50 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 06:57:04PM -0400, Robert Haas wrote:
> On Wed, Mar 23, 2022 at 5:52 PM Justin Pryzby <[email protected]> wrote:
> > Also because the library may not be compiled with threading. A few days ago, I
> > tried to rebase the original "parallel workers" patch over the COMPRESS DETAIL
> > patch but then couldn't test it, even after trying various versions of the zstd
> > package and trying to compile it locally. I'll try again soon...
>
> Ah. Right, I can update the comment to mention that.
Actually, I suggest to remove those comments:
| "We check for failure here because..."
That should be the rule rather than the exception, so shouldn't require
justifying why one might checks the return value of library and system calls.
In bbsink_zstd_new(), I think you need to check to see if workers were
requested (same as the issue you found with "level"). If someone builds
against a version of zstd which doesn't support some parameter, you'll
currently call SetParameter with that flag anyway, with a default value.
That's not currently breaking anything for me (even though workers=N doesn't
work) but I think it's fragile and could break, maybe when compiled against an
old zstd, or with future options. SetParameter should only be called when the
user requested to set the parameter. I handled that for workers in 003, but
didn't touch "level", which is probably fine, but maybe should change for
consistency.
src/backend/replication/basebackup_zstd.c: elog(ERROR, "could not set zstd compression level to %d: %s",
src/bin/pg_basebackup/bbstreamer_gzip.c: pg_log_error("could not set compression level %d: %s",
src/bin/pg_basebackup/bbstreamer_zstd.c: pg_log_error("could not set compression level to: %d: %s",
I'm not sure why these messages sometimes mention the current compression
method and sometimes don't. I suggest that they shouldn't - errcontext will
have the algorithm, and the user already specified it anyway. It'd allow the
compiler to merge strings.
Here's a patch for zstd --long mode. (I don't actually use pg_basebackup, but
I will want to use long mode with pg_dump). The "strategy" params may also be
interesting, but I haven't played with it. rsyncable is certainly interesting,
but currently an experimental, nonpublic interface - and a good example of why
to not call SetParameter for params which the user didn't specify: PGDG might
eventually compile postgres against a zstd which supports rsyncable flag. And
someone might install somewhere which doesn't support rsyncable, but the server
would try to call SetParameter(rsyncable, 0), and the rsyncable ID number
would've changed, so zstd would probably reject it, and basebackup would be
unusable...
$ time src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method=none --no-manifest -Z zstd:long=1 --checkpoint fast |wc -c
4625935
real 0m1,334s
$ time src/bin/pg_basebackup/pg_basebackup -h /tmp -Ft -D- --wal-method=none --no-manifest -Z zstd:long=0 --checkpoint fast |wc -c
8426516
real 0m0,880s
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 22:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-27 20:50 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
@ 2022-03-28 16:57 ` Robert Haas <[email protected]>
2022-03-28 17:32 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-28 16:57 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Sun, Mar 27, 2022 at 4:50 PM Justin Pryzby <[email protected]> wrote:
> Actually, I suggest to remove those comments:
> | "We check for failure here because..."
>
> That should be the rule rather than the exception, so shouldn't require
> justifying why one might checks the return value of library and system calls.
I went for modifying the comment rather than removing it. I agree with
you that checking for failure doesn't really require justification,
but I think that in a case like this it is useful to explain what we
know about why it might fail.
> In bbsink_zstd_new(), I think you need to check to see if workers were
> requested (same as the issue you found with "level").
Fixed.
> src/backend/replication/basebackup_zstd.c: elog(ERROR, "could not set zstd compression level to %d: %s",
> src/bin/pg_basebackup/bbstreamer_gzip.c: pg_log_error("could not set compression level %d: %s",
> src/bin/pg_basebackup/bbstreamer_zstd.c: pg_log_error("could not set compression level to: %d: %s",
>
> I'm not sure why these messages sometimes mention the current compression
> method and sometimes don't. I suggest that they shouldn't - errcontext will
> have the algorithm, and the user already specified it anyway. It'd allow the
> compiler to merge strings.
I don't think that errcontext() helps here. On the client side, it
doesn't exist. On the server side, it's not in use. I do see
STATEMENT: <whatever> in the server log when a replication command
throws a server-side error, which is similar, but pg_basebackup
doesn't display that STATEMENT line. I don't really know how to
balance the legitimate desire for future messages against the
also-legitimate desire for clarity about where things are failing. I'm
slightly inclined to think that including the algorithm name is
better, because options are in the end algorithm-specific, but it's
certainly debatable. I would be interested in hearing other
opinions...
Here's an updated and rebased version of my patch.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v2-0001-Allow-parallel-zstd-compression-when-taking-a-bas.patch (12.5K, ../../CA+TgmoauS-Sq1dwPAQQ2f-zKvPh27zrUaN0qqKOC+8cSsqQz4Q@mail.gmail.com/2-v2-0001-Allow-parallel-zstd-compression-when-taking-a-bas.patch)
download | inline diff:
From 473e410a7625fe3fb84a34eab594a84fd40bd2a7 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Wed, 23 Mar 2022 11:00:33 -0400
Subject: [PATCH v2] Allow parallel zstd compression when taking a base backup.
libzstd allows transparent parallel compression just by setting
an option when creating the compression context, so permit that
for both client and server-side backup compression. To use this,
use something like pg_basebackup --compress WHERE-zstd:workers=N
where WHERE is "client" or "server" and N is an integer.
When compression is performed on the server side, this will spawn
threads inside the PostgreSQL backend. While there is almost no
PostgreSQL server code which is thread-safe, the threads here are used
internally by libzstd and touch only data structures controlled by
libzstd.
Patch by me, based in part on earlier work by Dipesh Pandit
and Jeevan Ladhe.
---
doc/src/sgml/protocol.sgml | 12 +++++--
doc/src/sgml/ref/pg_basebackup.sgml | 4 +--
src/backend/replication/basebackup_zstd.c | 18 ++++++++++
src/bin/pg_basebackup/bbstreamer_zstd.c | 17 ++++++++++
src/bin/pg_basebackup/t/010_pg_basebackup.pl | 5 +++
src/bin/pg_verifybackup/t/009_extract.pl | 29 ++++++++++++++--
src/bin/pg_verifybackup/t/010_client_untar.pl | 33 +++++++++++++++++--
src/common/backup_compression.c | 16 +++++++++
src/include/common/backup_compression.h | 2 ++
src/test/perl/PostgreSQL/Test/Cluster.pm | 3 +-
10 files changed, 127 insertions(+), 12 deletions(-)
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 2fa3cedfe9..98f0bc3cc3 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2739,17 +2739,23 @@ The commands accepted in replication mode are:
option. If the value is an integer, it specifies the compression
level. Otherwise, it should be a comma-separated list of items,
each of the form <literal>keyword</literal> or
- <literal>keyword=value</literal>. Currently, the only supported
- keyword is <literal>level</literal>, which sets the compression
- level.
+ <literal>keyword=value</literal>. Currently, the supported keywords
+ are <literal>level</literal> and <literal>workers</literal>.
</para>
<para>
+ The <literal>level</literal> keyword sets the compression level.
For <literal>gzip</literal> the compression level should be an
integer between 1 and 9, for <literal>lz4</literal> an integer
between 1 and 12, and for <literal>zstd</literal> an integer
between 1 and 22.
</para>
+
+ <para>
+ The <literal>workers</literal> keyword sets the number of threads
+ that should be used for parallel compression. Parallel compression
+ is supported only for <literal>zstd</literal>.
+ </para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index d9233beb8e..82f5f60625 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -424,8 +424,8 @@ PostgreSQL documentation
integer, it specifies the compression level. Otherwise, it should be
a comma-separated list of items, each of the form
<literal>keyword</literal> or <literal>keyword=value</literal>.
- Currently, the only supported keyword is <literal>level</literal>,
- which sets the compression level.
+ Currently, the supported keywords are <literal>level</literal>
+ and <literal>workers</literal>.
</para>
<para>
If no compression level is specified, the default compression level
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index 5496eaa72b..d6eb0617d8 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -28,6 +28,9 @@ typedef struct bbsink_zstd
/* Compression level */
int compresslevel;
+ /* Number of parallel workers. */
+ int workers;
+
ZSTD_CCtx *cctx;
ZSTD_outBuffer zstd_outBuf;
} bbsink_zstd;
@@ -83,6 +86,7 @@ bbsink_zstd_new(bbsink *next, bc_specification *compress)
*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
sink->base.bbs_next = next;
sink->compresslevel = compresslevel;
+ sink->workers = compress->workers;
return &sink->base;
#endif
@@ -110,6 +114,20 @@ bbsink_zstd_begin_backup(bbsink *sink)
elog(ERROR, "could not set zstd compression level to %d: %s",
mysink->compresslevel, ZSTD_getErrorName(ret));
+ /*
+ * We check for failure here because (1) older versions of the library
+ * do not support ZSTD_c_nbWorkers and (2) the library might want to
+ * reject an unreasonable values (though in practice it does not seem to do
+ * so).
+ */
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_nbWorkers,
+ mysink->workers);
+ if (ZSTD_isError(ret))
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not set compression worker count to %d: %s",
+ mysink->workers, ZSTD_getErrorName(ret)));
+
/*
* We need our own buffer, because we're going to pass different data to
* the next sink than what gets passed to us.
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index 7946b6350b..50bae5d4be 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -102,6 +102,23 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
exit(1);
}
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0)
+ {
+ /*
+ * On older versions of libzstd, this option does not exist, and
+ * trying to set it will fail. Similarly for newer versions if they
+ * are compiled without threading support.
+ */
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
+ compress->workers);
+ if (ZSTD_isError(ret))
+ {
+ pg_log_error("could not set compression worker count to %d: %s",
+ compress->workers, ZSTD_getErrorName(ret));
+ exit(1);
+ }
+ }
+
/* Initialize the ZSTD output buffer. */
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 47f3d00ac4..5ba84c2250 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -130,6 +130,11 @@ my @compression_failure_tests = (
'invalid compression specification: found empty string where a compression option was expected',
'failure on extra, empty compression option'
],
+ [
+ 'gzip:workers=3',
+ 'invalid compression specification: compression algorithm "gzip" does not accept a worker count',
+ 'failure on worker count for gzip'
+ ],
);
for my $cft (@compression_failure_tests)
{
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index 41a5b370cc..d6f11b9553 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -34,6 +34,12 @@ my @test_configuration = (
'compression_method' => 'zstd',
'backup_flags' => ['--compress', 'server-zstd:5'],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'server-zstd:workers=3'],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -55,8 +61,27 @@ for my $tc (@test_configuration)
my @verify = ('pg_verifybackup', '-e', $backup_path);
# A backup with a valid compression method should work.
- $primary->command_ok(\@backup,
- "backup done, compression method \"$method\"");
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 2;
+ }
+ else
+ {
+ ok($backup_result, "backup done, compression $method");
+ }
# Make sure that it verifies OK.
$primary->command_ok(\@verify,
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 488a6d1ede..c1cd12cb06 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -49,6 +49,15 @@ my @test_configuration = (
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:workers=3'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -69,9 +78,27 @@ for my $tc (@test_configuration)
'pg_basebackup', '-D', $backup_path,
'-Xfetch', '--no-sync', '-cfast', '-Ft');
push @backup, @{$tc->{'backup_flags'}};
- $primary->command_ok(\@backup,
- "client side backup, compression $method");
-
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 3;
+ }
+ else
+ {
+ ok($backup_result, "client side backup, compression $method");
+ }
# Verify that the we got the files we expected.
my $backup_files = join(',',
diff --git a/src/common/backup_compression.c b/src/common/backup_compression.c
index 0650f975c4..969e08cca2 100644
--- a/src/common/backup_compression.c
+++ b/src/common/backup_compression.c
@@ -177,6 +177,11 @@ parse_bc_specification(bc_algorithm algorithm, char *specification,
result->level = expect_integer_value(keyword, value, result);
result->options |= BACKUP_COMPRESSION_OPTION_LEVEL;
}
+ else if (strcmp(keyword, "workers") == 0)
+ {
+ result->workers = expect_integer_value(keyword, value, result);
+ result->options |= BACKUP_COMPRESSION_OPTION_WORKERS;
+ }
else
result->parse_error =
psprintf(_("unknown compression option \"%s\""), keyword);
@@ -266,5 +271,16 @@ validate_bc_specification(bc_specification *spec)
min_level, max_level);
}
+ /*
+ * Of the compression algorithms that we currently support, only zstd
+ * allows parallel workers.
+ */
+ if ((spec->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0 &&
+ (spec->algorithm != BACKUP_COMPRESSION_ZSTD))
+ {
+ return psprintf(_("compression algorithm \"%s\" does not accept a worker count"),
+ get_bc_algorithm_name(spec->algorithm));
+ }
+
return NULL;
}
diff --git a/src/include/common/backup_compression.h b/src/include/common/backup_compression.h
index 0565cbc657..6a0ecaa99c 100644
--- a/src/include/common/backup_compression.h
+++ b/src/include/common/backup_compression.h
@@ -23,12 +23,14 @@ typedef enum bc_algorithm
} bc_algorithm;
#define BACKUP_COMPRESSION_OPTION_LEVEL (1 << 0)
+#define BACKUP_COMPRESSION_OPTION_WORKERS (1 << 1)
typedef struct bc_specification
{
bc_algorithm algorithm;
unsigned options; /* OR of BACKUP_COMPRESSION_OPTION constants */
int level;
+ int workers;
char *parse_error; /* NULL if parsing was OK, else message */
} bc_specification;
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index bee6aacf47..b6e3351611 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2502,8 +2502,7 @@ sub run_log
local %ENV = $self->_get_env();
- PostgreSQL::Test::Utils::run_log(@_);
- return;
+ return PostgreSQL::Test::Utils::run_log(@_);
}
=pod
--
2.24.3 (Apple Git-128)
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 22:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-27 20:50 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-28 16:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-28 17:32 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-28 17:32 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Mon, Mar 28, 2022 at 12:57 PM Robert Haas <[email protected]> wrote:
> Here's an updated and rebased version of my patch.
Well, that only updated the comment on the client side. Let's try again.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v3-0001-Allow-parallel-zstd-compression-when-taking-a-bas.patch (14.6K, ../../CA+TgmoZ-0qNqkUVr9wb1NfQ3=fB4JVN3=OZ9yMSuarcd44hYAQ@mail.gmail.com/2-v3-0001-Allow-parallel-zstd-compression-when-taking-a-bas.patch)
download | inline diff:
From 29ae6c4909e0c3ce3f66f869f06278fb109749f4 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Mon, 28 Mar 2022 13:25:44 -0400
Subject: [PATCH v3] Allow parallel zstd compression when taking a base backup.
libzstd allows transparent parallel compression just by setting
an option when creating the compression context, so permit that
for both client and server-side backup compression. To use this,
use something like pg_basebackup --compress WHERE-zstd:workers=N
where WHERE is "client" or "server" and N is an integer.
When compression is performed on the server side, this will spawn
threads inside the PostgreSQL backend. While there is almost no
PostgreSQL server code which is thread-safe, the threads here are used
internally by libzstd and touch only data structures controlled by
libzstd.
Patch by me, based in part on earlier work by Dipesh Pandit
and Jeevan Ladhe. Reviewed by Justin Pryzby.
---
doc/src/sgml/protocol.sgml | 12 +++--
doc/src/sgml/ref/pg_basebackup.sgml | 4 +-
src/backend/replication/basebackup_zstd.c | 45 ++++++++++++-------
src/bin/pg_basebackup/bbstreamer_zstd.c | 40 ++++++++++++-----
src/bin/pg_basebackup/t/010_pg_basebackup.pl | 5 +++
src/bin/pg_verifybackup/t/009_extract.pl | 29 +++++++++++-
src/bin/pg_verifybackup/t/010_client_untar.pl | 33 ++++++++++++--
src/common/backup_compression.c | 16 +++++++
src/include/common/backup_compression.h | 2 +
src/test/perl/PostgreSQL/Test/Cluster.pm | 3 +-
10 files changed, 148 insertions(+), 41 deletions(-)
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 2fa3cedfe9..98f0bc3cc3 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2739,17 +2739,23 @@ The commands accepted in replication mode are:
option. If the value is an integer, it specifies the compression
level. Otherwise, it should be a comma-separated list of items,
each of the form <literal>keyword</literal> or
- <literal>keyword=value</literal>. Currently, the only supported
- keyword is <literal>level</literal>, which sets the compression
- level.
+ <literal>keyword=value</literal>. Currently, the supported keywords
+ are <literal>level</literal> and <literal>workers</literal>.
</para>
<para>
+ The <literal>level</literal> keyword sets the compression level.
For <literal>gzip</literal> the compression level should be an
integer between 1 and 9, for <literal>lz4</literal> an integer
between 1 and 12, and for <literal>zstd</literal> an integer
between 1 and 22.
</para>
+
+ <para>
+ The <literal>workers</literal> keyword sets the number of threads
+ that should be used for parallel compression. Parallel compression
+ is supported only for <literal>zstd</literal>.
+ </para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index d9233beb8e..82f5f60625 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -424,8 +424,8 @@ PostgreSQL documentation
integer, it specifies the compression level. Otherwise, it should be
a comma-separated list of items, each of the form
<literal>keyword</literal> or <literal>keyword=value</literal>.
- Currently, the only supported keyword is <literal>level</literal>,
- which sets the compression level.
+ Currently, the supported keywords are <literal>level</literal>
+ and <literal>workers</literal>.
</para>
<para>
If no compression level is specified, the default compression level
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index 5496eaa72b..f6876f4811 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -25,8 +25,8 @@ typedef struct bbsink_zstd
/* Common information for all types of sink. */
bbsink base;
- /* Compression level */
- int compresslevel;
+ /* Compression options */
+ bc_specification *compress;
ZSTD_CCtx *cctx;
ZSTD_outBuffer zstd_outBuf;
@@ -67,22 +67,13 @@ bbsink_zstd_new(bbsink *next, bc_specification *compress)
return NULL; /* keep compiler quiet */
#else
bbsink_zstd *sink;
- int compresslevel;
Assert(next != NULL);
- if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0)
- compresslevel = 0;
- else
- {
- compresslevel = compress->level;
- Assert(compresslevel >= 1 && compresslevel <= 22);
- }
-
sink = palloc0(sizeof(bbsink_zstd));
*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops;
sink->base.bbs_next = next;
- sink->compresslevel = compresslevel;
+ sink->compress = compress;
return &sink->base;
#endif
@@ -99,16 +90,36 @@ bbsink_zstd_begin_backup(bbsink *sink)
bbsink_zstd *mysink = (bbsink_zstd *) sink;
size_t output_buffer_bound;
size_t ret;
+ bc_specification *compress = mysink->compress;
mysink->cctx = ZSTD_createCCtx();
if (!mysink->cctx)
elog(ERROR, "could not create zstd compression context");
- ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
- mysink->compresslevel);
- if (ZSTD_isError(ret))
- elog(ERROR, "could not set zstd compression level to %d: %s",
- mysink->compresslevel, ZSTD_getErrorName(ret));
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0)
+ {
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
+ compress->level);
+ if (ZSTD_isError(ret))
+ elog(ERROR, "could not set zstd compression level to %d: %s",
+ compress->level, ZSTD_getErrorName(ret));
+ }
+
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0)
+ {
+ /*
+ * On older versions of libzstd, this option does not exist, and trying
+ * to set it will fail. Similarly for newer versions if they are
+ * compiled without threading support.
+ */
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_nbWorkers,
+ compress->workers);
+ if (ZSTD_isError(ret))
+ ereport(ERROR,
+ errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not set compression worker count to %d: %s",
+ compress->workers, ZSTD_getErrorName(ret)));
+ }
/*
* We need our own buffer, because we're going to pass different data to
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index 7946b6350b..f94c5c041d 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -67,7 +67,6 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
{
#ifdef USE_ZSTD
bbstreamer_zstd_frame *streamer;
- int compresslevel;
size_t ret;
Assert(next != NULL);
@@ -88,18 +87,35 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
exit(1);
}
- /* Initialize stream compression preferences */
- if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0)
- compresslevel = 0;
- else
- compresslevel = compress->level;
- ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
- compresslevel);
- if (ZSTD_isError(ret))
+ /* Set compression level, if specified */
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0)
{
- pg_log_error("could not set zstd compression level to %d: %s",
- compresslevel, ZSTD_getErrorName(ret));
- exit(1);
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
+ compress->level);
+ if (ZSTD_isError(ret))
+ {
+ pg_log_error("could not set zstd compression level to %d: %s",
+ compress->level, ZSTD_getErrorName(ret));
+ exit(1);
+ }
+ }
+
+ /* Set # of workers, if specified */
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0)
+ {
+ /*
+ * On older versions of libzstd, this option does not exist, and
+ * trying to set it will fail. Similarly for newer versions if they
+ * are compiled without threading support.
+ */
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
+ compress->workers);
+ if (ZSTD_isError(ret))
+ {
+ pg_log_error("could not set compression worker count to %d: %s",
+ compress->workers, ZSTD_getErrorName(ret));
+ exit(1);
+ }
}
/* Initialize the ZSTD output buffer. */
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 47f3d00ac4..5ba84c2250 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -130,6 +130,11 @@ my @compression_failure_tests = (
'invalid compression specification: found empty string where a compression option was expected',
'failure on extra, empty compression option'
],
+ [
+ 'gzip:workers=3',
+ 'invalid compression specification: compression algorithm "gzip" does not accept a worker count',
+ 'failure on worker count for gzip'
+ ],
);
for my $cft (@compression_failure_tests)
{
diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index 41a5b370cc..d6f11b9553 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -34,6 +34,12 @@ my @test_configuration = (
'compression_method' => 'zstd',
'backup_flags' => ['--compress', 'server-zstd:5'],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'server-zstd:workers=3'],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -55,8 +61,27 @@ for my $tc (@test_configuration)
my @verify = ('pg_verifybackup', '-e', $backup_path);
# A backup with a valid compression method should work.
- $primary->command_ok(\@backup,
- "backup done, compression method \"$method\"");
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 2;
+ }
+ else
+ {
+ ok($backup_result, "backup done, compression $method");
+ }
# Make sure that it verifies OK.
$primary->command_ok(\@verify,
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
index 488a6d1ede..c1cd12cb06 100644
--- a/src/bin/pg_verifybackup/t/010_client_untar.pl
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -49,6 +49,15 @@ my @test_configuration = (
'decompress_program' => $ENV{'ZSTD'},
'decompress_flags' => [ '-d' ],
'enabled' => check_pg_config("#define USE_ZSTD 1")
+ },
+ {
+ 'compression_method' => 'parallel zstd',
+ 'backup_flags' => ['--compress', 'client-zstd:workers=3'],
+ 'backup_archive' => 'base.tar.zst',
+ 'decompress_program' => $ENV{'ZSTD'},
+ 'decompress_flags' => [ '-d' ],
+ 'enabled' => check_pg_config("#define USE_ZSTD 1"),
+ 'possibly_unsupported' => qr/could not set compression worker count to 3: Unsupported parameter/
}
);
@@ -69,9 +78,27 @@ for my $tc (@test_configuration)
'pg_basebackup', '-D', $backup_path,
'-Xfetch', '--no-sync', '-cfast', '-Ft');
push @backup, @{$tc->{'backup_flags'}};
- $primary->command_ok(\@backup,
- "client side backup, compression $method");
-
+ my $backup_stdout = '';
+ my $backup_stderr = '';
+ my $backup_result = $primary->run_log(\@backup, '>', \$backup_stdout,
+ '2>', \$backup_stderr);
+ if ($backup_stdout ne '')
+ {
+ print "# standard output was:\n$backup_stdout";
+ }
+ if ($backup_stderr ne '')
+ {
+ print "# standard error was:\n$backup_stderr";
+ }
+ if (! $backup_result && $tc->{'possibly_unsupported'} &&
+ $backup_stderr =~ /$tc->{'possibly_unsupported'}/)
+ {
+ skip "compression with $method not supported by this build", 3;
+ }
+ else
+ {
+ ok($backup_result, "client side backup, compression $method");
+ }
# Verify that the we got the files we expected.
my $backup_files = join(',',
diff --git a/src/common/backup_compression.c b/src/common/backup_compression.c
index 0650f975c4..969e08cca2 100644
--- a/src/common/backup_compression.c
+++ b/src/common/backup_compression.c
@@ -177,6 +177,11 @@ parse_bc_specification(bc_algorithm algorithm, char *specification,
result->level = expect_integer_value(keyword, value, result);
result->options |= BACKUP_COMPRESSION_OPTION_LEVEL;
}
+ else if (strcmp(keyword, "workers") == 0)
+ {
+ result->workers = expect_integer_value(keyword, value, result);
+ result->options |= BACKUP_COMPRESSION_OPTION_WORKERS;
+ }
else
result->parse_error =
psprintf(_("unknown compression option \"%s\""), keyword);
@@ -266,5 +271,16 @@ validate_bc_specification(bc_specification *spec)
min_level, max_level);
}
+ /*
+ * Of the compression algorithms that we currently support, only zstd
+ * allows parallel workers.
+ */
+ if ((spec->options & BACKUP_COMPRESSION_OPTION_WORKERS) != 0 &&
+ (spec->algorithm != BACKUP_COMPRESSION_ZSTD))
+ {
+ return psprintf(_("compression algorithm \"%s\" does not accept a worker count"),
+ get_bc_algorithm_name(spec->algorithm));
+ }
+
return NULL;
}
diff --git a/src/include/common/backup_compression.h b/src/include/common/backup_compression.h
index 0565cbc657..6a0ecaa99c 100644
--- a/src/include/common/backup_compression.h
+++ b/src/include/common/backup_compression.h
@@ -23,12 +23,14 @@ typedef enum bc_algorithm
} bc_algorithm;
#define BACKUP_COMPRESSION_OPTION_LEVEL (1 << 0)
+#define BACKUP_COMPRESSION_OPTION_WORKERS (1 << 1)
typedef struct bc_specification
{
bc_algorithm algorithm;
unsigned options; /* OR of BACKUP_COMPRESSION_OPTION constants */
int level;
+ int workers;
char *parse_error; /* NULL if parsing was OK, else message */
} bc_specification;
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index bee6aacf47..b6e3351611 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2502,8 +2502,7 @@ sub run_log
local %ENV = $self->_get_env();
- PostgreSQL::Test::Utils::run_log(@_);
- return;
+ return PostgreSQL::Test::Utils::run_log(@_);
}
=pod
--
2.24.3 (Apple Git-128)
^ permalink raw reply [nested|flat] 103+ messages in thread
* fixing a few backup compression goofs
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
@ 2022-03-24 21:56 ` Robert Haas <[email protected]>
2022-03-25 13:23 ` Re: fixing a few backup compression goofs Dipesh Pandit <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-24 21:56 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 5:52 PM Justin Pryzby <[email protected]> wrote:
> I think you should also test the return value when setting the compress level.
> Not only because it's generally a good idea, but also because I suggested to
> support negative compression levels. Which weren't allowed before v1.3.4, and
> then the range is only defined since 1.3.6 (ZSTD_minCLevel). At some point,
> the range may have been -7..22 but now it's -131072..22.
Hi,
The attached patch fixes a few goofs around backup compression. It
adds a check that setting the compression level succeeds, although it
does not allow the broader range of compression levels Justin notes
above. That can be done separately, I guess, if we want to do it. It
also fixes the problem that client and server-side zstd compression
don't actually compress equally well; that turned out to be a bug in
the handling of compression options. Finally it adds an exit call to
an unlikely failure case so that we would, if that case should occur,
print a message and exit, rather than the current behavior of printing
a message and then dereferencing a null pointer.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] 0001-Fix-a-few-goofs-in-new-backup-compression-code.patch (4.4K, ../../CA+TgmoZK3zLQUCGi1h4XZw4jHiAWtcACc+GsdJR1_Mc19jUjXA@mail.gmail.com/2-0001-Fix-a-few-goofs-in-new-backup-compression-code.patch)
download | inline diff:
From 7c04715c6f5410e3be4f62c29edef60401d721a9 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Thu, 24 Mar 2022 17:21:11 -0400
Subject: [PATCH] Fix a few goofs in new backup compression code.
When we try to set the zstd compression level either on the client
or on the server, check for errors.
For any algorithm, on the client side, don't try to set the compression
level unless the user specified one. This was visibly broken for
zstd, which managed to set -1 rather than 0 in this case, but tidy
up the code for the other methods, too.
On the client side, if we fail to create a ZSTD_CCtx, exit after
reporting the error. Otherwise we'll dereference a null pointer.
---
src/backend/replication/basebackup_zstd.c | 8 ++++++--
src/bin/pg_basebackup/bbstreamer_gzip.c | 3 ++-
src/bin/pg_basebackup/bbstreamer_lz4.c | 3 ++-
src/bin/pg_basebackup/bbstreamer_zstd.c | 19 +++++++++++++++++--
4 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c
index bb5b668c2a..5496eaa72b 100644
--- a/src/backend/replication/basebackup_zstd.c
+++ b/src/backend/replication/basebackup_zstd.c
@@ -98,13 +98,17 @@ bbsink_zstd_begin_backup(bbsink *sink)
{
bbsink_zstd *mysink = (bbsink_zstd *) sink;
size_t output_buffer_bound;
+ size_t ret;
mysink->cctx = ZSTD_createCCtx();
if (!mysink->cctx)
elog(ERROR, "could not create zstd compression context");
- ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
- mysink->compresslevel);
+ ret = ZSTD_CCtx_setParameter(mysink->cctx, ZSTD_c_compressionLevel,
+ mysink->compresslevel);
+ if (ZSTD_isError(ret))
+ elog(ERROR, "could not set zstd compression level to %d: %s",
+ mysink->compresslevel, ZSTD_getErrorName(ret));
/*
* We need our own buffer, because we're going to pass different data to
diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/bbstreamer_gzip.c
index 1979e95639..760619fcd7 100644
--- a/src/bin/pg_basebackup/bbstreamer_gzip.c
+++ b/src/bin/pg_basebackup/bbstreamer_gzip.c
@@ -116,7 +116,8 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file,
}
}
- if (gzsetparams(streamer->gzfile, compress->level,
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0 &&
+ gzsetparams(streamer->gzfile, compress->level,
Z_DEFAULT_STRATEGY) != Z_OK)
{
pg_log_error("could not set compression level %d: %s",
diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c
index a6ec317e2b..67f841d96a 100644
--- a/src/bin/pg_basebackup/bbstreamer_lz4.c
+++ b/src/bin/pg_basebackup/bbstreamer_lz4.c
@@ -89,7 +89,8 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, bc_specification *compress)
prefs = &streamer->prefs;
memset(prefs, 0, sizeof(LZ4F_preferences_t));
prefs->frameInfo.blockSizeID = LZ4F_max256KB;
- prefs->compressionLevel = compress->level;
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0)
+ prefs->compressionLevel = compress->level;
/*
* Find out the compression bound, it specifies the minimum destination
diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c
index caa5edcaf1..7946b6350b 100644
--- a/src/bin/pg_basebackup/bbstreamer_zstd.c
+++ b/src/bin/pg_basebackup/bbstreamer_zstd.c
@@ -67,6 +67,8 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
{
#ifdef USE_ZSTD
bbstreamer_zstd_frame *streamer;
+ int compresslevel;
+ size_t ret;
Assert(next != NULL);
@@ -81,11 +83,24 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress)
streamer->cctx = ZSTD_createCCtx();
if (!streamer->cctx)
+ {
pg_log_error("could not create zstd compression context");
+ exit(1);
+ }
/* Initialize stream compression preferences */
- ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
- compress->level);
+ if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0)
+ compresslevel = 0;
+ else
+ compresslevel = compress->level;
+ ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
+ compresslevel);
+ if (ZSTD_isError(ret))
+ {
+ pg_log_error("could not set zstd compression level to %d: %s",
+ compresslevel, ZSTD_getErrorName(ret));
+ exit(1);
+ }
/* Initialize the ZSTD output buffer. */
streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
--
2.24.3 (Apple Git-128)
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: fixing a few backup compression goofs
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-24 21:56 ` fixing a few backup compression goofs Robert Haas <[email protected]>
@ 2022-03-25 13:23 ` Dipesh Pandit <[email protected]>
2022-03-28 16:28 ` Re: fixing a few backup compression goofs Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Dipesh Pandit @ 2022-03-25 13:23 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
Hi,
The changes look good to me.
Thanks,
Dipesh
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: fixing a few backup compression goofs
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-24 21:56 ` fixing a few backup compression goofs Robert Haas <[email protected]>
2022-03-25 13:23 ` Re: fixing a few backup compression goofs Dipesh Pandit <[email protected]>
@ 2022-03-28 16:28 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-28 16:28 UTC (permalink / raw)
To: Dipesh Pandit <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Fri, Mar 25, 2022 at 9:23 AM Dipesh Pandit <[email protected]> wrote:
> The changes look good to me.
Thanks. Committed.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-23 23:07 ` Justin Pryzby <[email protected]>
2022-03-23 23:36 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-24 13:00 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
4 siblings, 2 replies; 103+ messages in thread
From: Justin Pryzby @ 2022-03-23 23:07 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 04:34:04PM -0400, Robert Haas wrote:
> be, spawning threads inside the PostgreSQL backend. Short of cats and
> dogs living together, it's hard to think of anything more terrifying,
> because the PostgreSQL backend is very much not thread-safe. However,
> a lot of the things we usually worry about when people make noises
> about using threads in the backend don't apply here, because the
> threads are hidden away behind libzstd interfaces and can't execute
> any PostgreSQL code. Therefore, I think it might be safe to just ...
> turn this on. One reason I think that is that this whole approach was
> recommended to me by Andres ... but that's not to say that there
> couldn't be problems. I worry a bit that the mere presence of threads
> could in some way mess things up, but I don't know what the mechanism
> for that would be, and I don't want to postpone shipping useful
> features based on nebulous fears.
Note that the PGDG .RPMs and .DEBs are already linked with pthread, via
libxml => liblzma.
$ ldd /usr/pgsql-14/bin/postgres |grep xm
libxml2.so.2 => /lib64/libxml2.so.2 (0x00007faab984e000)
$ objdump -p /lib64/libxml2.so.2 |grep NEED
NEEDED libdl.so.2
NEEDED libz.so.1
NEEDED liblzma.so.5
NEEDED libm.so.6
NEEDED libc.so.6
VERNEED 0x0000000000019218
VERNEEDNUM 0x0000000000000005
$ objdump -p /lib64/liblzma.so.5 |grep NEED
NEEDED libpthread.so.0
Did you try this on windows at all ? It's probably no surprise that zstd
implements threading differently there.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 23:07 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
@ 2022-03-23 23:36 ` Andres Freund <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Andres Freund @ 2022-03-23 23:36 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Robert Haas <[email protected]>; Dipesh Pandit <[email protected]>; pgsql-hackers
On 2022-03-23 18:07:01 -0500, Justin Pryzby wrote:
> Did you try this on windows at all ?
Really should get zstd installed in the windows cf environment...
> It's probably no surprise that zstd implements threading differently there.
Worth noting that we have a few of our own threads running on windows already
- so we're guaranteed to build against the threaded standard libraries etc
already.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 23:07 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
@ 2022-03-24 13:00 ` Robert Haas <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Robert Haas @ 2022-03-24 13:00 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 7:07 PM Justin Pryzby <[email protected]> wrote:
> Did you try this on windows at all ? It's probably no surprise that zstd
> implements threading differently there.
I did not. I haven't had a properly functioning Windows development
environment in about a decade.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-24 13:19 ` Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
4 siblings, 1 reply; 103+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-03-24 13:19 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Hi Robert,
I haven't reviewed the meat of the patch in detail, but I noticed
something in the tests:
Robert Haas <[email protected]> writes:
> diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
> index 9f9cc7540b..e17e7cad51 100644
> --- a/src/bin/pg_verifybackup/t/009_extract.pl
> +++ b/src/bin/pg_verifybackup/t/009_extract.pl
[…]
> + if ($backup_stdout ne '')
> + {
> + print "# standard output was:\n$backup_stdout";
> + }
> + if ($backup_stderr ne '')
> + {
> + print "# standard error was:\n$backup_stderr";
> + }
[…]
> diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
> index 487e30e826..5f6a4b9963 100644
> --- a/src/bin/pg_verifybackup/t/010_client_untar.pl
> +++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
[…]
> + if ($backup_stdout ne '')
> + {
> + print "# standard output was:\n$backup_stdout";
> + }
> + if ($backup_stderr ne '')
> + {
> + print "# standard error was:\n$backup_stderr";
> + }
Per the TAP protocol, every line of non-test-result output should be
prefixed by "# ". The note() function does this for you, see
https://metacpan.org/pod/Test::More#Diagnostics for details.
- ilmari
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
@ 2022-03-28 16:37 ` Robert Haas <[email protected]>
2022-03-28 16:52 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-28 16:37 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Thu, Mar 24, 2022 at 9:19 AM Dagfinn Ilmari Mannsåker
<[email protected]> wrote:
> Per the TAP protocol, every line of non-test-result output should be
> prefixed by "# ". The note() function does this for you, see
> https://metacpan.org/pod/Test::More#Diagnostics for details.
True, but that also means it shows up in the actual failure message,
which seems too verbose. By just using 'print', it ends up in the log
file if it's needed, but not anywhere else. Maybe there's a better way
to do this, but I don't think using note() is what I want.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-28 16:52 ` Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:55 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-03-28 16:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Robert Haas <[email protected]> writes:
> On Thu, Mar 24, 2022 at 9:19 AM Dagfinn Ilmari Mannsåker
> <[email protected]> wrote:
>> Per the TAP protocol, every line of non-test-result output should be
>> prefixed by "# ". The note() function does this for you, see
>> https://metacpan.org/pod/Test::More#Diagnostics for details.
>
> True, but that also means it shows up in the actual failure message,
> which seems too verbose. By just using 'print', it ends up in the log
> file if it's needed, but not anywhere else. Maybe there's a better way
> to do this, but I don't think using note() is what I want.
That is the difference between note() and diag(): note() prints to
stdout so is not visible under a non-verbose prove run, while diag()
prints to stderr so it's always visible.
- ilmari
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-28 16:52 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
@ 2022-03-28 16:55 ` Robert Haas <[email protected]>
2022-03-30 12:06 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-28 16:55 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Mon, Mar 28, 2022 at 12:52 PM Dagfinn Ilmari Mannsåker
<[email protected]> wrote:
> > True, but that also means it shows up in the actual failure message,
> > which seems too verbose. By just using 'print', it ends up in the log
> > file if it's needed, but not anywhere else. Maybe there's a better way
> > to do this, but I don't think using note() is what I want.
>
> That is the difference between note() and diag(): note() prints to
> stdout so is not visible under a non-verbose prove run, while diag()
> prints to stderr so it's always visible.
OK, but print doesn't do either of those things. The output only shows
up in the log file, even with --verbose. Here's an example of what the
log file looks like:
# Running: pg_verifybackup -n -m
/Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/server-backup/backup_manifest
-e /Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/extracted-backup
backup successfully verified
ok 6 - verify backup, compression gzip
As you can see, there is a line here that does not begin with #. That
line is the standard output of a command that was run by the test
script.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-28 16:52 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:55 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-30 12:06 ` Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 12:55 ` Re: multithreaded zstd backup compression for client and server Andrew Dunstan <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-03-30 12:06 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Robert Haas <[email protected]> writes:
> On Mon, Mar 28, 2022 at 12:52 PM Dagfinn Ilmari Mannsåker
> <[email protected]> wrote:
>> > True, but that also means it shows up in the actual failure message,
>> > which seems too verbose. By just using 'print', it ends up in the log
>> > file if it's needed, but not anywhere else. Maybe there's a better way
>> > to do this, but I don't think using note() is what I want.
>>
>> That is the difference between note() and diag(): note() prints to
>> stdout so is not visible under a non-verbose prove run, while diag()
>> prints to stderr so it's always visible.
>
> OK, but print doesn't do either of those things. The output only shows
> up in the log file, even with --verbose. Here's an example of what the
> log file looks like:
>
> # Running: pg_verifybackup -n -m
> /Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/server-backup/backup_manifest
> -e /Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/extracted-backup
> backup successfully verified
> ok 6 - verify backup, compression gzip
>
> As you can see, there is a line here that does not begin with #. That
> line is the standard output of a command that was run by the test
> script.
Oh, that must be some non-standard output handling that our test setup
does. Plain `prove` shows everything on stdout and stderr in verbose
mode, and only stderr in non-vebose mode:
$ cat verbosity.t
use strict;
use warnings;
use Test::More;
pass "pass";
diag "diag";
note "note";
print "print\n";
system qw(echo system);
done_testing;
$ prove verbosity.t
verbosity.t .. 1/? # diag
verbosity.t .. ok
All tests successful.
Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.04 cusr 0.01 csys = 0.07 CPU)
Result: PASS
$ prove -v verbosity.t
verbosity.t ..
ok 1 - pass
# diag
# note
print
system
1..1
ok
All tests successful.
Files=1, Tests=1, 0 wallclock secs ( 0.02 usr 0.00 sys + 0.06 cusr 0.00 csys = 0.08 CPU)
Result: PASS
- ilmari
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-28 16:52 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:55 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 12:06 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
@ 2022-03-30 12:55 ` Andrew Dunstan <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Andrew Dunstan @ 2022-03-30 12:55 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On 3/30/22 08:06, Dagfinn Ilmari Mannsåker wrote:
> Robert Haas <[email protected]> writes:
>
>> On Mon, Mar 28, 2022 at 12:52 PM Dagfinn Ilmari Mannsåker
>> <[email protected]> wrote:
>>>> True, but that also means it shows up in the actual failure message,
>>>> which seems too verbose. By just using 'print', it ends up in the log
>>>> file if it's needed, but not anywhere else. Maybe there's a better way
>>>> to do this, but I don't think using note() is what I want.
>>> That is the difference between note() and diag(): note() prints to
>>> stdout so is not visible under a non-verbose prove run, while diag()
>>> prints to stderr so it's always visible.
>> OK, but print doesn't do either of those things. The output only shows
>> up in the log file, even with --verbose. Here's an example of what the
>> log file looks like:
>>
>> # Running: pg_verifybackup -n -m
>> /Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/server-backup/backup_manifest
>> -e /Users/rhaas/pgsql/src/bin/pg_verifybackup/tmp_check/t_008_untar_primary_data/backup/extracted-backup
>> backup successfully verified
>> ok 6 - verify backup, compression gzip
>>
>> As you can see, there is a line here that does not begin with #. That
>> line is the standard output of a command that was run by the test
>> script.
> Oh, that must be some non-standard output handling that our test setup
> does. Plain `prove` shows everything on stdout and stderr in verbose
> mode, and only stderr in non-vebose mode:
>
Yes, PostgreSQL::Test::Utils hijacks STDOUT and STDERR (see the INIT
block).
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-30 12:00 ` Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 15:44 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-31 12:21 ` Re: multithreaded zstd backup compression for client and server Andrew Dunstan <[email protected]>
4 siblings, 2 replies; 103+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-03-30 12:00 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Robert Haas <[email protected]> writes:
> This patch contains a trivial adjustment to
> PostgreSQL::Test::Cluster::run_log to make it return a useful value
> instead of not. I think that should be pulled out and committed
> independently regardless of what happens to this patch overall, and
> possibly back-patched.
run_log() is far from the only such method in PostgreSQL::Test::Cluster.
Here's a patch that gives the same treatment to all the methods that
just pass through to the corresponding PostgreSQL::Test::Utils function.
Also attached is a fix a typo in the _get_env doc comment that I noticed
while auditing the return values.
- ilmari
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 12:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
@ 2022-03-30 15:44 ` Robert Haas <[email protected]>
2022-03-30 17:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
1 sibling, 1 reply; 103+ messages in thread
From: Robert Haas @ 2022-03-30 15:44 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Wed, Mar 30, 2022 at 8:00 AM Dagfinn Ilmari Mannsåker
<[email protected]> wrote:
> Robert Haas <[email protected]> writes:
> > This patch contains a trivial adjustment to
> > PostgreSQL::Test::Cluster::run_log to make it return a useful value
> > instead of not. I think that should be pulled out and committed
> > independently regardless of what happens to this patch overall, and
> > possibly back-patched.
>
> run_log() is far from the only such method in PostgreSQL::Test::Cluster.
> Here's a patch that gives the same treatment to all the methods that
> just pass through to the corresponding PostgreSQL::Test::Utils function.
>
> Also attached is a fix a typo in the _get_env doc comment that I noticed
> while auditing the return values.
I suggest posting these patches on a new thread with a subject line
that matches what the patches do, and adding it to the next
CommitFest. It seems like a reasonable thing to do on first glance,
but I wouldn't want to commit it without going through and figuring
out whether there's any risk of anything breaking, and it doesn't seem
like there's a strong need to do it in v15 rather than v16.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 12:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 15:44 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
@ 2022-03-30 17:00 ` Dagfinn Ilmari Mannsåker <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-03-30 17:00 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Robert Haas <[email protected]> writes:
> On Wed, Mar 30, 2022 at 8:00 AM Dagfinn Ilmari Mannsåker
> <[email protected]> wrote:
>> Robert Haas <[email protected]> writes:
>> > This patch contains a trivial adjustment to
>> > PostgreSQL::Test::Cluster::run_log to make it return a useful value
>> > instead of not. I think that should be pulled out and committed
>> > independently regardless of what happens to this patch overall, and
>> > possibly back-patched.
>>
>> run_log() is far from the only such method in PostgreSQL::Test::Cluster.
>> Here's a patch that gives the same treatment to all the methods that
>> just pass through to the corresponding PostgreSQL::Test::Utils function.
>>
>> Also attached is a fix a typo in the _get_env doc comment that I noticed
>> while auditing the return values.
>
> I suggest posting these patches on a new thread with a subject line
> that matches what the patches do, and adding it to the next
> CommitFest.
Will do.
> It seems like a reasonable thing to do on first glance, but I wouldn't
> want to commit it without going through and figuring out whether
> there's any risk of anything breaking, and it doesn't seem like
> there's a strong need to do it in v15 rather than v16.
Given that the methods don't currently have a useful return value (undef
or the empty list, depending on context), I don't expect anything to be
relying on it (and it passed check-world with --enable-tap-tests and all
the --with-foo flags I could easily get to work), but I can grep the
code as well to be extra sure.
- ilmari
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: multithreaded zstd backup compression for client and server
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 12:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
@ 2022-03-31 12:21 ` Andrew Dunstan <[email protected]>
1 sibling, 0 replies; 103+ messages in thread
From: Andrew Dunstan @ 2022-03-31 12:21 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; Robert Haas <[email protected]>; +Cc: Dipesh Pandit <[email protected]>; Andres Freund <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On 3/30/22 08:00, Dagfinn Ilmari Mannsåker wrote:
> Robert Haas <[email protected]> writes:
>
>> This patch contains a trivial adjustment to
>> PostgreSQL::Test::Cluster::run_log to make it return a useful value
>> instead of not. I think that should be pulled out and committed
>> independently regardless of what happens to this patch overall, and
>> possibly back-patched.
> run_log() is far from the only such method in PostgreSQL::Test::Cluster.
> Here's a patch that gives the same treatment to all the methods that
> just pass through to the corresponding PostgreSQL::Test::Utils function.
>
> Also attached is a fix a typo in the _get_env doc comment that I noticed
> while auditing the return values.
>
None of these routines in Utils.pm returns a useful value (unlike
run_log()). Typically we don't return the value of Test::More routines.
So -1 on patch 1. I will fix the typo.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
@ 2022-03-30 13:44 Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2022-03-30 13:44 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]
On Tue, Mar 01, 2022 at 02:14:13PM +0900, Kyotaro Horiguchi wrote:
> Rebased on a recent xlog refactoring.
It'll come as no surprise that this neds to be rebased again.
At least a few typos I reported in January aren't fixed.
Set to "waiting".
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
@ 2022-03-31 04:58 ` Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-03-31 04:58 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]
Thanks! Version 20 is attached.
At Wed, 30 Mar 2022 08:44:02 -0500, Justin Pryzby <[email protected]> wrote in
> On Tue, Mar 01, 2022 at 02:14:13PM +0900, Kyotaro Horiguchi wrote:
> > Rebased on a recent xlog refactoring.
>
> It'll come as no surprise that this neds to be rebased again.
> At least a few typos I reported in January aren't fixed.
> Set to "waiting".
Oh, I'm sorry for overlooking it. It somehow didn't show up on my
mailer.
> I started looking at this and reviewed docs and comments again.
>
> > +typedef struct PendingCleanup
> > +{
> > + RelFileNode relnode; /* relation that may need to be deleted */
> > + int op; /* operation mask */
> > + bool bufpersistence; /* buffer persistence to set */
> > + int unlink_forknum; /* forknum to unlink */
>
> This can be of data type "ForkNumber"
Right. Fixed.
> > + * We are going to create an init fork. If server crashes before the
> > + * current transaction ends the init fork left alone corrupts data while
> > + * recovery. The mark file works as the sentinel to identify that
> > + * situation.
>
> s/while/during/
This was in v17, but dissapeared in v18.
> > + * index-init fork needs further initialization. ambuildempty shoud do
>
> should (I reported this before)
>
> > + if (inxact_created)
> > + {
> > + SMgrRelation srel = smgropen(rnode, InvalidBackendId);
> > +
> > + /*
> > + * INIT forks never be loaded to shared buffer so no point in dropping
>
> "are never loaded"
If was fixed in v18.
> > + elog(DEBUG1, "perform in-place persistnce change");
>
> persistence (I reported this before)
Sorry. Fixed.
> > + /*
> > + * While wal_level >= replica, switching to LOGGED requires the
> > + * relation content to be WAL-logged to recover the table.
> > + * We don't emit this fhile wal_level = minimal.
>
> while (or "if")
There are "While" and "fhile". I changed the latter to "if".
> > + * The relation is persistent and stays remain persistent. Don't
> > + * drop the buffers for this relation.
>
> "stays remain" is redundant (I reported this before)
Thanks. I changed it to "stays persistent".
> > + if (unlink(rm_path) < 0)
> > + ereport(ERROR,
> > + (errcode_for_file_access(),
> > + errmsg("could not remove file \"%s\": %m",
> > + rm_path)));
>
> The parens around errcode are unnecessary since last year.
> I suggest to avoid using them here and elsewhere.
It is just moved from elsewhere without editing, but of course I can
do that. I didn't know about that change of ereport and not found the
corresponding commit, but I found that Tom mentioned that change.
https://www.postgresql.org/message-id/flat/5063.1584641224%40sss.pgh.pa.us#63e611c30800133bbddb48de8...
> Now that we can rely on having varargs macros, I think we could
> stop requiring the extra level of parentheses, ie instead of
...
> ereport(ERROR,
> errcode(ERRCODE_DIVISION_BY_ZERO),
> errmsg("division by zero"));
>
> (The old syntax had better still work, of course. I'm not advocating
> running around and changing existing calls.)
I changed all ereport calls added by this patch to this style.
> > + * And we may have SMGR_MARK_UNCOMMITTED file. Remove it if the
> > + * fork files has been successfully removed. It's ok if the file
>
> file
Fixed.
> > + <para>
> > + All tables in the current database in a tablespace can be changed by using
>
> given tablespace
I did /database in a tablespace/database in the given tablespace/. Is
it right?
> > + the <literal>ALL IN TABLESPACE</literal> form, which will lock all tables
>
> which will first lock
>
> > + to be changed first and then change each one. This form also supports
>
> remove "first" here
This is almost a dead copy of the description of SET TABLESPACE. This
change makes the two almost the same description vary slightly in that
wordings. Anyway I did that as suggested only for the part this patch
adds in this version.
> > + <literal>OWNED BY</literal>, which will only change tables owned by the
> > + roles specified. If the <literal>NOWAIT</literal> option is specified
>
> specified roles.
> is specified, (comma)
This is the same as above. I did that but it makes the description
differ from another almost-the-same description.
> > + then the command will fail if it is unable to acquire all of the locks
>
> if it is unable to immediately acquire
>
> > + required immediately. The <literal>information_schema</literal>
>
> remove immediately
Ditto.
> > + relations are not considered part of the system catalogs and will be
>
> I think you need to first say that "relations in the pg_catalog schema cannot
> be changed".
Mmm. I don't agree on this. Aren't such "exceptions"-ish descriptions
usually placed after the descriptions of how the feature works? This
is also the same structure with SET TABLESPACE.
> in patch 2/2:
> typo: persistene
Hmm. Bad. I checked the spellings of the whole patches and found some
typos.
+ * The crashed trasaction did SET UNLOGGED. This relation
+ * is restored to a LOGGED relation.
s/trasaction/transaction/
+ errmsg("could not crete mark file \"%s\": %m", path));
s/crete/create/
Then rebased on 9c08aea6a3 then pgindent'ed.
Thanks!
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-03-31 05:37 ` Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Justin Pryzby @ 2022-03-31 05:37 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]
On Thu, Mar 31, 2022 at 01:58:45PM +0900, Kyotaro Horiguchi wrote:
> Thanks! Version 20 is attached.
The patch failed an all CI tasks, and seems to have caused the macos task to
hang.
http://cfbot.cputube.org/kyotaro-horiguchi.html
Would you send a fixed patch, or remove this thread from the CFBOT ? Otherwise
cirrrus will try to every day to rerun but take 1hr to time out, which is twice
as slow as the slowest OS.
I think this patch should be moved to the next CF and set to v16.
Thanks,
--
Justin
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
@ 2022-03-31 09:33 ` Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-03-31 09:33 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]
At Thu, 31 Mar 2022 00:37:07 -0500, Justin Pryzby <[email protected]> wrote in
> On Thu, Mar 31, 2022 at 01:58:45PM +0900, Kyotaro Horiguchi wrote:
> > Thanks! Version 20 is attached.
>
> The patch failed an all CI tasks, and seems to have caused the macos task to
> hang.
>
> http://cfbot.cputube.org/kyotaro-horiguchi.html
>
> Would you send a fixed patch, or remove this thread from the CFBOT ? Otherwis
e
> cirrrus will try to every day to rerun but take 1hr to time out, which is twice
> as slow as the slowest OS.
That is found to be a thinko that causes mark files left behind in new
database created in the logged version of CREATE DATABASE. It is
easily fixed.
That being said, this failure revealed that pg_checksums or
pg_basebackup dislikes the mark files. It happens even in a quite low
possibility. This would need further consideration and extra rounds of
reviews.
> I think this patch should be moved to the next CF and set to v16.
I don't think this can be commited to 15. So I post the fixed version
then move this to the next CF.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-03-31 09:36 ` Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-03-31 09:36 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]
At Thu, 31 Mar 2022 18:33:18 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> I don't think this can be commited to 15. So I post the fixed version
> then move this to the next CF.
Then done. Thanks!
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-07-06 15:44 ` Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Jacob Champion @ 2022-07-06 15:44 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Tom Lane <[email protected]>; [email protected]; [email protected]; PostgreSQL Hackers <[email protected]>
On Thu, Mar 31, 2022 at 2:36 AM Kyotaro Horiguchi
<[email protected]> wrote:
>
> At Thu, 31 Mar 2022 18:33:18 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > I don't think this can be commited to 15. So I post the fixed version
> > then move this to the next CF.
>
> Then done. Thanks!
Hello! This patchset will need to be rebased over latest -- looks like
b74e94dc27f (Rethink PROCSIGNAL_BARRIER_SMGRRELEASE) and 5c279a6d350
(Custom WAL Resource Managers) are interfering.
Thanks,
--Jacob
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
@ 2022-07-07 08:24 ` Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-07-07 08:24 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Wed, 6 Jul 2022 08:44:18 -0700, Jacob Champion <[email protected]> wrote in
> On Thu, Mar 31, 2022 at 2:36 AM Kyotaro Horiguchi
> <[email protected]> wrote:
> >
> > At Thu, 31 Mar 2022 18:33:18 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> > > I don't think this can be commited to 15. So I post the fixed version
> > > then move this to the next CF.
> >
> > Then done. Thanks!
>
> Hello! This patchset will need to be rebased over latest -- looks like
> b74e94dc27f (Rethink PROCSIGNAL_BARRIER_SMGRRELEASE) and 5c279a6d350
> (Custom WAL Resource Managers) are interfering.
Thank you for checking that! It got a wider attack by b0a55e4329
(RelFileNumber). The commit message suggests "relfilenode" as files
should be replaced with "relation storage/file" so I did that in
ResetUnloggedRelationsInDbspaceDir.
This patch said that:
> * INIT forks are never loaded to shared buffer so no point in
> * dropping buffers for such files.
But actually some *buildempty() functions use ReadBufferExtended() for
INIT_FORK. So that's wrong. So, I did that but... I don't like that.
Or I don't like that some AMs leave buffers for INIT fork after. But I
feel I'm misunderstanding here since I don't understand how the INIT
fork can work as expected after a crash that happens before the next
checkpoint flushes the buffers.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-07-19 04:33 ` Kyotaro Horiguchi <[email protected]>
2022-09-28 08:21 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-07-19 04:33 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
(Mmm. I haven't noticed an annoying misspelling in the subejct X( )
> Thank you for checking that! It got a wider attack by b0a55e4329
> (RelFileNumber). The commit message suggests "relfilenode" as files
Then, now I stepped on my own foot. Rebased also on nodefuncs
autogeneration.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-09-28 08:21 ` Kyotaro Horiguchi <[email protected]>
2022-11-04 00:32 ` Re: In-placre persistance change of a relation Ian Lawrence Barwick <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-09-28 08:21 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Just rebased.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-09-28 08:21 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-11-04 00:32 ` Ian Lawrence Barwick <[email protected]>
2022-11-08 02:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Ian Lawrence Barwick @ 2022-11-04 00:32 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
2022年9月28日(水) 17:21 Kyotaro Horiguchi <[email protected]>:
>
> Just rebased.
Hi
cfbot reports the patch no longer applies. As CommitFest 2022-11 is
currently underway, this would be an excellent time to update the patch.
Thanks
Ian Barwick
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-09-28 08:21 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-11-04 00:32 ` Re: In-placre persistance change of a relation Ian Lawrence Barwick <[email protected]>
@ 2022-11-08 02:33 ` Kyotaro Horiguchi <[email protected]>
2022-11-18 08:25 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-11-08 02:33 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Fri, 4 Nov 2022 09:32:52 +0900, Ian Lawrence Barwick <[email protected]> wrote in
> cfbot reports the patch no longer applies. As CommitFest 2022-11 is
> currently underway, this would be an excellent time to update the patch.
Indeed, thanks! I'll do that in a few days.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-09-28 08:21 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-11-04 00:32 ` Re: In-placre persistance change of a relation Ian Lawrence Barwick <[email protected]>
2022-11-08 02:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
@ 2022-11-18 08:25 ` Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2022-11-18 08:25 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Tue, 08 Nov 2022 11:33:53 +0900 (JST), Kyotaro Horiguchi <[email protected]> wrote in
> Indeed, thanks! I'll do that in a few days.
Got too late, but rebased.. The contents of the two patches in the
last version was a bit shuffled but they are fixed.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
@ 2024-01-22 04:36 Peter Smith <[email protected]>
2024-01-23 04:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 103+ messages in thread
From: Peter Smith @ 2024-01-22 04:36 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
2024-01 Commitfest.
Hi, This patch has a CF status of "Needs Review" [1], but it seems
there was a CFbot test failure last time it was run [2]. Please have a
look and post an updated version if necessary.
======
[1] https://commitfest.postgresql.org/46/3461/
[2] https://cirrus-ci.com/task/6050020441456640
Kind Regards,
Peter Smith.
^ permalink raw reply [nested|flat] 103+ messages in thread
* Re: In-placre persistance change of a relation
2024-01-22 04:36 Re: In-placre persistance change of a relation Peter Smith <[email protected]>
@ 2024-01-23 04:15 ` Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 103+ messages in thread
From: Kyotaro Horiguchi @ 2024-01-23 04:15 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
At Mon, 22 Jan 2024 15:36:31 +1100, Peter Smith <[email protected]> wrote in
> 2024-01 Commitfest.
>
> Hi, This patch has a CF status of "Needs Review" [1], but it seems
> there was a CFbot test failure last time it was run [2]. Please have a
> look and post an updated version if necessary.
Thanks! I have added the necessary includes to the header file this
patch adds. With this change, "make headerscheck" now passes. However,
when I run "make cpluspluscheck" in my environment, it generates a
large number of errors in other areas, but I didn't find one related
to this patch.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 103+ messages in thread
end of thread, other threads:[~2024-01-23 04:15 UTC | newest]
Thread overview: 103+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2018-08-15 08:34 [PATCH 2/5] Quick-Vacuum-Strategy Andrey V. Lepikhov <[email protected]>
2020-11-11 08:33 In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-11 22:18 ` Re: In-placre persistance change of a relation Andres Freund <[email protected]>
2020-11-12 06:55 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 04:22 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 06:43 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 07:15 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-11-13 08:23 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-04 07:49 ` RE: In-placre persistance change of a relation [email protected] <[email protected]>
2020-12-24 08:02 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-12-25 00:12 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 05:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-08 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-12 09:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-01-14 08:32 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:08 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-03-25 05:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-17 09:10 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-17 12:46 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2021-12-17 14:33 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-17 19:10 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2021-12-20 06:28 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:53 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 07:59 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-20 08:39 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 08:52 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-20 13:38 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-21 08:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 11:04 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-21 13:07 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-22 06:13 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-22 08:42 ` RE: In-placre persistance change of a relation Jakub Wartak <[email protected]>
2021-12-23 06:01 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2021-12-23 06:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2020-11-13 07:47 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-01-20 07:36 [PATCH v13 05/20] meson: prereq: Add src/tools/gen_export.pl Andres Freund <[email protected]>
2022-01-20 07:36 [PATCH v11 4/9] meson: prereq: Add src/tools/gen_export.pl Andres Freund <[email protected]>
2022-01-20 07:36 [PATCH v13 05/20] meson: prereq: Add src/tools/gen_export.pl Andres Freund <[email protected]>
2022-01-20 07:36 [PATCH v12 04/15] meson: prereq: Add src/tools/gen_export.pl Andres Freund <[email protected]>
2022-02-16 21:07 Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-02-17 01:46 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-04 08:31 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-04 14:31 ` walmethods.c is kind of a mess (was Re: refactoring basebackup.c) Robert Haas <[email protected]>
2022-03-07 21:25 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 09:49 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 15:28 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:32 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-08 16:53 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-08 16:58 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-11 01:02 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-11 15:19 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-11 16:29 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2022-03-12 01:52 ` Re: refactoring basebackup.c Andres Freund <[email protected]>
2022-03-14 13:27 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-14 16:11 ` Re: refactoring basebackup.c Dipesh Pandit <[email protected]>
2022-03-14 16:35 ` Re: refactoring basebackup.c (zstd workers) Justin Pryzby <[email protected]>
2022-03-15 10:33 ` Re: refactoring basebackup.c Jeevan Ladhe <[email protected]>
2022-03-15 17:50 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-23 02:08 ` Re: refactoring basebackup.c Thomas Munro <[email protected]>
2023-03-23 20:11 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-03-24 14:46 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2023-04-12 14:57 ` Re: refactoring basebackup.c Justin Pryzby <[email protected]>
2023-04-12 15:55 ` Re: refactoring basebackup.c Robert Haas <[email protected]>
2022-03-23 20:34 ` multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:14 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-23 22:31 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 23:31 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-24 13:39 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-23 21:52 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 22:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-27 20:50 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-28 16:57 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-28 17:32 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 21:56 ` fixing a few backup compression goofs Robert Haas <[email protected]>
2022-03-25 13:23 ` Re: fixing a few backup compression goofs Dipesh Pandit <[email protected]>
2022-03-28 16:28 ` Re: fixing a few backup compression goofs Robert Haas <[email protected]>
2022-03-23 23:07 ` Re: multithreaded zstd backup compression for client and server Justin Pryzby <[email protected]>
2022-03-23 23:36 ` Re: multithreaded zstd backup compression for client and server Andres Freund <[email protected]>
2022-03-24 13:00 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-24 13:19 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:37 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-28 16:52 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-28 16:55 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 12:06 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 12:55 ` Re: multithreaded zstd backup compression for client and server Andrew Dunstan <[email protected]>
2022-03-30 12:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-30 15:44 ` Re: multithreaded zstd backup compression for client and server Robert Haas <[email protected]>
2022-03-30 17:00 ` Re: multithreaded zstd backup compression for client and server Dagfinn Ilmari Mannsåker <[email protected]>
2022-03-31 12:21 ` Re: multithreaded zstd backup compression for client and server Andrew Dunstan <[email protected]>
2022-03-30 13:44 Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 04:58 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 05:37 ` Re: In-placre persistance change of a relation Justin Pryzby <[email protected]>
2022-03-31 09:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-03-31 09:36 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-06 15:44 ` Re: In-placre persistance change of a relation Jacob Champion <[email protected]>
2022-07-07 08:24 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-07-19 04:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-09-28 08:21 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-11-04 00:32 ` Re: In-placre persistance change of a relation Ian Lawrence Barwick <[email protected]>
2022-11-08 02:33 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2022-11-18 08:25 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
2024-01-22 04:36 Re: In-placre persistance change of a relation Peter Smith <[email protected]>
2024-01-23 04:15 ` Re: In-placre persistance change of a relation Kyotaro Horiguchi <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox