agora inbox for [email protected]
help / color / mirror / Atom feedFrom: Kyotaro Horiguchi <[email protected]>
Subject: [PATCH v36 14/17] In-place persistance change to UNLOGGED
Date: Tue, 27 Aug 2024 11:19:53 +0900
This commit enables changing the persistence of relations to UNLOGGED
without creating a new storage file. ALTER TABLE LOGGED will continue
to create a new storage as before.
---
src/backend/access/transam/undolog.c | 32 +++-
src/backend/access/transam/xact.c | 8 +-
src/backend/catalog/storage.c | 112 ++++++++++++-
src/backend/commands/tablecmds.c | 226 ++++++++++++++++++++++-----
src/include/access/undolog.h | 2 +-
5 files changed, 331 insertions(+), 49 deletions(-)
diff --git a/src/backend/access/transam/undolog.c b/src/backend/access/transam/undolog.c
index b2fdbfcd0f9..33d1b35ae8d 100644
--- a/src/backend/access/transam/undolog.c
+++ b/src/backend/access/transam/undolog.c
@@ -757,9 +757,10 @@ UndoLog_UndoByXid(bool isCommit, TransactionId xid,
* During recovery, it should pass the target transaction ID.
*/
void
-AtEOXact_UndoLog(TransactionId xid)
+AtEOXact_UndoLog(bool isCommit, TransactionId xid)
{
FullTransactionId fxid = ULogLocal.current_xid;
+ bool redo = false;
if (TransactionIdIsValid(xid))
{
@@ -767,7 +768,8 @@ AtEOXact_UndoLog(TransactionId xid)
TransactionId oldest_xid;
TransactionId next_xid;
uint32 oldest_epoch;
-
+
+ redo = true;
LWLockAcquire(XactTruncationLock, LW_SHARED);
next_fxid = TransamVariables->nextXid;
oldest_xid = TransamVariables->oldestClogXid;
@@ -785,7 +787,31 @@ AtEOXact_UndoLog(TransactionId xid)
}
if (FullTransactionIdIsValid(fxid))
- undolog_drop_ulog(fxid);
+ {
+ UndoLogSlot *slot;
+
+ slot = undolog_find_slot(fxid, false);
+ if (slot)
+ {
+ undolog_flush_slot(slot, false);
+ LWLockRelease(&slot->lock);
+ }
+
+ if (slot || undolog_file_exists(fxid))
+ {
+ char fname[MAXPGPATH];
+ ULogContext cxt;
+
+ if (isCommit)
+ cxt = ULOGCXT_COMMIT;
+ else
+ cxt = ULOGCXT_ABORT;
+
+ UndoLogSetFilename(fname, fxid);
+ undolog_process_ulog(fname, cxt, redo);
+ undolog_drop_ulog(fxid);
+ }
+ }
}
/*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index b609a783464..197fde27edc 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -2480,7 +2480,7 @@ CommitTransaction(void)
AtEOXact_on_commit_actions(true);
AtEOXact_Namespace(true, is_parallel_worker);
AtEOXact_SMgr();
- AtEOXact_UndoLog(InvalidTransactionId);
+ AtEOXact_UndoLog(true, InvalidTransactionId);
AtEOXact_Files(true);
AtEOXact_ComboCid();
AtEOXact_HashTables(true);
@@ -2999,7 +2999,7 @@ AbortTransaction(void)
AtEOXact_on_commit_actions(false);
AtEOXact_Namespace(false, is_parallel_worker);
AtEOXact_SMgr();
- AtEOXact_UndoLog(InvalidTransactionId);
+ AtEOXact_UndoLog(false, InvalidTransactionId);
AtEOXact_Files(false);
AtEOXact_ComboCid();
AtEOXact_HashTables(false);
@@ -6269,7 +6269,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed,
true);
}
- AtEOXact_UndoLog(xid);
+ AtEOXact_UndoLog(true, xid);
AtEOXact_Buffers_Redo(true, xid, parsed->nsubxacts, parsed->subxacts);
if (parsed->nstats > 0)
@@ -6384,7 +6384,7 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid,
true);
}
- AtEOXact_UndoLog(xid);
+ AtEOXact_UndoLog(false, xid);
AtEOXact_Buffers_Redo(false, xid, parsed->nsubxacts, parsed->subxacts);
if (parsed->nstats > 0)
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index 67f2b3727a9..ad8855b69b4 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -32,6 +32,8 @@
#include "miscadmin.h"
#include "storage/bulk_write.h"
#include "storage/freespace.h"
+#include "storage/copydir.h"
+#include "storage/fd.h"
#include "storage/proc.h"
#include "storage/smgr.h"
#include "utils/hsearch.h"
@@ -558,6 +560,58 @@ RelationTruncate(Relation rel, BlockNumber nblocks)
FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
}
+/*
+ * Reset an unlogged relation using the INIT fork, intended for use during the
+ * commit of prepared transactions. The relation is assumed to be UNLOGGED, so
+ * no WAL-logging is required.
+ */
+static void
+ResetUnloggedRelation(RelFileLocator rloc, ProcNumber backend)
+{
+ char *srcpath;
+ char *dstpath;
+ SMgrRelation srel = smgropen(rloc, backend);
+ ForkNumber forks[MAX_FORKNUM];
+ BlockNumber blocks[MAX_FORKNUM];
+ BlockNumber old_blocks[MAX_FORKNUM];
+ int nforks = 0;
+
+ srel = smgropen(rloc, backend);
+
+ Assert(smgrexists(srel, INIT_FORKNUM));
+
+ for (int i = 0 ; i <= MAX_FORKNUM ; i++)
+ {
+ if (i == INIT_FORKNUM || !smgrexists(srel, i))
+ continue;
+
+ forks[nforks] = i;
+ old_blocks[nforks] = smgrnblocks(srel, i);
+ blocks[nforks] = 0;
+ nforks++;
+ }
+
+ /*
+ * This relation is unlogged. Therefore, unlike RelationTruncate(), there
+ * is no need to call RelationPreTruncate().
+ */
+ START_CRIT_SECTION();
+ smgrtruncate(srel, forks, nforks, old_blocks, blocks);
+ END_CRIT_SECTION();
+
+ /* Note that this leaves the first segment of the main fork. */
+ for (int i = 0 ; i < nforks ; i++)
+ smgrunlink(srel, forks[i], false);
+
+ /* copy init fork to main fork */
+ srcpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber,
+ backend, INIT_FORKNUM);
+ dstpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber,
+ backend, MAIN_FORKNUM);
+ copy_file_extended(srcpath, dstpath, true);
+ fsync_fname(dstpath, false);
+}
+
/*
* RelationPreTruncate
* Perform AM-independent work before a physical truncation.
@@ -1314,8 +1368,62 @@ smgr_undo(UndoLogRecord *record, ULogContext cxt, bool redo, bool crashed)
else
elog(PANIC, "smgr_undo: unknown op code %d", info);
}
- else if (cxt == ULOGCXT_COMMIT || cxt == ULOGCXT_ABORT ||
- cxt == ULOGCXT_PREPARED)
+ else if (cxt == ULOGCXT_COMMIT)
+ {
+ Assert(record);
+ info = record->ul_info & ~ULR_INFO_MASK;
+
+ if (info == ULOG_SMGR_CREATE)
+ {
+ ul_smgr_create *ulrec = (ul_smgr_create *) ULogRecGetData(record);
+ /*
+ * If an init fork was created during recovery, the entire relation
+ * is set to be reset at recovery-end or the consistency point.
+ * Therefore, we need to drop the relation's buffers to prevent the
+ * end-of-recovery checkpoint from flushing storage files for these
+ * relations once they have been reset.
+ */
+ if (redo && ulrec->forknum == INIT_FORKNUM)
+ {
+ SMgrRelation reln;
+ int nforks;
+ ForkNumber forks[MAX_FORKNUM + 1];
+ BlockNumber firstblocks[MAX_FORKNUM + 1] = {0};
+
+ Assert(ulrec->backend == INVALID_PROC_NUMBER);
+
+ reln = smgropen(ulrec->rlocator, ulrec->backend);
+
+ nforks = 0;
+ for (int i = 0 ; i <= MAX_FORKNUM ; i++)
+ {
+ if (smgrexists(reln, i))
+ forks[nforks++] = i;
+ }
+
+ if (nforks > 0)
+ DropRelationBuffers(reln, forks, nforks, firstblocks);
+
+ smgrclose(reln);
+ }
+ else if (!redo && crashed && ulrec->forknum == INIT_FORKNUM)
+ {
+ /*
+ * System has been crashed until the transaction was
+ * prepared. Now that the init fork is persists, the relation
+ * needs to be cleared.
+ */
+ ResetUnloggedRelation(ulrec->rlocator, ulrec->backend);
+ ereport(WARNING,
+ errmsg("unlogged relation %u/%u/%u was reset",
+ ulrec->rlocator.spcOid, ulrec->rlocator.dbOid,
+ ulrec->rlocator.relNumber),
+ errdetail("Server experinced a crash after the transaction that altered the relation was prepared."));
+ }
+
+ }
+ }
+ else if(cxt == ULOGCXT_PREPARED || cxt == ULOGCXT_ABORT)
{
/* nothing to do here */
}
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index b5766989d8e..3c58d5be464 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -5744,6 +5744,143 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel,
return newcmd;
}
+/*
+ * RelationChangePersistence: perform in-place persistence change of a relation
+ */
+static void
+RelationChangePersistence(AlteredTableInfo *tab, char persistence,
+ LOCKMODE lockmode)
+{
+ Relation rel;
+ Relation classRel;
+ HeapTuple tuple,
+ newtuple;
+ Datum new_val[Natts_pg_class];
+ bool new_null[Natts_pg_class],
+ new_repl[Natts_pg_class];
+ List *relids;
+ ListCell *lc_oid;
+
+ Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE);
+ Assert(lockmode == AccessExclusiveLock);
+
+ /*
+ * Use ATRewriteTable instead of this function if the following condition
+ * is not satisfied.
+ */
+ Assert(tab->constraints == NULL && tab->partition_constraint == NULL &&
+ tab->newvals == NULL && !tab->verify_new_notnull);
+
+ rel = table_open(tab->relid, lockmode);
+
+ Assert(rel->rd_rel->relpersistence != persistence);
+
+ elog(DEBUG1, "perform in-place persistence change");
+
+ /*
+ * Initially, gather all relations that require a persistence change.
+ */
+
+ /* Collect OIDs of indexes and toast relations */
+ relids = RelationGetIndexList(rel);
+ relids = lcons_oid(rel->rd_id, relids);
+
+ /* Add toast relation if any */
+ if (OidIsValid(rel->rd_rel->reltoastrelid))
+ {
+ List *toastidx;
+ Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode);
+
+ relids = lappend_oid(relids, rel->rd_rel->reltoastrelid);
+ toastidx = RelationGetIndexList(toastrel);
+ relids = list_concat(relids, toastidx);
+ pfree(toastidx);
+ table_close(toastrel, NoLock);
+ }
+
+ table_close(rel, NoLock);
+
+ /* Make changes in storage */
+ classRel = table_open(RelationRelationId, RowExclusiveLock);
+
+ foreach(lc_oid, relids)
+ {
+ Oid reloid = lfirst_oid(lc_oid);
+ Relation r = relation_open(reloid, lockmode);
+ SMgrRelation srel;
+ bool persistent = (persistence == RELPERSISTENCE_PERMANENT);
+ bool is_index;
+
+ /*
+ * Reconstruct the storage when permanent and unlogged storage types
+ * are incompatible.
+ */
+ if (r->rd_rel->relkind == RELKIND_INDEX &&
+ !r->rd_indam->amunloggedstoragecompatible)
+ {
+ int reindex_flags;
+ ReindexParams params = {0};
+
+ /* reindex doesn't allow concurrent use of the index */
+ table_close(r, NoLock);
+
+ reindex_flags =
+ REINDEX_REL_SUPPRESS_INDEX_USE |
+ REINDEX_REL_CHECK_CONSTRAINTS;
+
+ /* Set the same persistence with the parent relation. */
+ if (persistent)
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
+ else
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED;
+
+ /* this doesn't fire REINDEX event triegger */
+ reindex_index(NULL, reloid, reindex_flags, persistence, ¶ms);
+
+ continue;
+ }
+
+ /* Currently, only allowing changes to UNLOGGED. */
+ Assert(!persistent);
+
+ RelationAssumePersistenceChange(r);
+
+ /* switch buffer persistence */
+ srel = RelationGetSmgr(r);
+ log_smgrbufpersistence(srel->smgr_rlocator.locator, persistent);
+ SetRelationBuffersPersistence(srel, persistent);
+
+ /* then create the init fork */
+ is_index = (r->rd_rel->relkind == RELKIND_INDEX);
+ RelationCreateFork(srel, INIT_FORKNUM, !is_index, true);
+ if (is_index)
+ r->rd_indam->ambuildempty(r);
+
+ /* Update catalog */
+ tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+ if (!HeapTupleIsValid(tuple))
+ elog(ERROR, "cache lookup failed for relation %u", reloid);
+
+ memset(new_val, 0, sizeof(new_val));
+ memset(new_null, false, sizeof(new_null));
+ memset(new_repl, false, sizeof(new_repl));
+
+ new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence);
+ new_null[Anum_pg_class_relpersistence - 1] = false;
+ new_repl[Anum_pg_class_relpersistence - 1] = true;
+
+ newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
+ new_val, new_null, new_repl);
+
+ CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+
+ table_close(r, NoLock);
+ }
+
+ table_close(classRel, NoLock);
+}
+
/*
* ATRewriteTables: ALTER TABLE phase 3
*/
@@ -5876,48 +6013,59 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode,
tab->relid,
tab->rewrite);
- /*
- * Create transient table that will receive the modified data.
- *
- * Ensure it is marked correctly as logged or unlogged. We have
- * to do this here so that buffers for the new relfilenumber will
- * have the right persistence set, and at the same time ensure
- * that the original filenumbers's buffers will get read in with
- * the correct setting (i.e. the original one). Otherwise a
- * rollback after the rewrite would possibly result with buffers
- * for the original filenumbers having the wrong persistence
- * setting.
- *
- * NB: This relies on swap_relation_files() also swapping the
- * persistence. That wouldn't work for pg_class, but that can't be
- * unlogged anyway.
- */
- OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod,
- persistence, lockmode);
+ if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE &&
+ persistence == RELPERSISTENCE_UNLOGGED)
+ {
+ /* Make in-place persistence change. */
+ RelationChangePersistence(tab, persistence, lockmode);
+ }
+ else
+ {
+ /*
+ * Create transient table that will receive the modified data.
+ *
+ * Ensure it is marked correctly as logged or unlogged. We
+ * have to do this here so that buffers for the new
+ * relfilenumber will have the right persistence set, and at
+ * the same time ensure that the original filenumbers's buffers
+ * will get read in with the correct setting (i.e. the original
+ * one). Otherwise a rollback after the rewrite would possibly
+ * result with buffers for the original filenumbers having the
+ * wrong persistence setting.
+ *
+ * NB: This relies on swap_relation_files() also swapping the
+ * persistence. That wouldn't work for pg_class, but that can't
+ * be unlogged anyway.
+ */
+ OIDNewHeap = make_new_heap(tab->relid, NewTableSpace,
+ NewAccessMethod,
+ persistence, lockmode);
- /*
- * Copy the heap data into the new table with the desired
- * modifications, and test the current data within the table
- * against new constraints generated by ALTER TABLE commands.
- */
- ATRewriteTable(tab, OIDNewHeap, lockmode);
+ /*
+ * Copy the heap data into the new table with the desired
+ * modifications, and test the current data within the table
+ * against new constraints generated by ALTER TABLE commands.
+ */
+ ATRewriteTable(tab, OIDNewHeap, lockmode);
- /*
- * Swap the physical files of the old and new heaps, then rebuild
- * indexes and discard the old heap. We can use RecentXmin for
- * the table's new relfrozenxid because we rewrote all the tuples
- * in ATRewriteTable, so no older Xid remains in the table. Also,
- * we never try to swap toast tables by content, since we have no
- * interest in letting this code work on system catalogs.
- */
- finish_heap_swap(tab->relid, OIDNewHeap,
- false, false, true,
- !OidIsValid(tab->newTableSpace),
- RecentXmin,
- ReadNextMultiXactId(),
- persistence);
+ /*
+ * Swap the physical files of the old and new heaps, then
+ * rebuild indexes and discard the old heap. We can use
+ * RecentXmin for the table's new relfrozenxid because we
+ * rewrote all the tuples in ATRewriteTable, so no older Xid
+ * remains in the table. Also, we never try to swap toast
+ * tables by content, since we have no interest in letting this
+ * code work on system catalogs.
+ */
+ finish_heap_swap(tab->relid, OIDNewHeap,
+ false, false, true,
+ !OidIsValid(tab->newTableSpace),
+ RecentXmin,
+ ReadNextMultiXactId(),
+ persistence);
- InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0);
+ InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0);
+ }
}
else if (tab->rewrite > 0 && tab->relkind == RELKIND_SEQUENCE)
{
diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h
index 19badc852a0..857a845eb9d 100644
--- a/src/include/access/undolog.h
+++ b/src/include/access/undolog.h
@@ -80,7 +80,7 @@ extern Size UndoLogShmemSize(void);
extern void UndoLogShmemInit(void);
extern void InitUndoLog(void);
extern void UndoLogWrite(RmgrId rmgr, uint8 info, void *data, int len);
-extern void AtEOXact_UndoLog(TransactionId xid);
+extern void AtEOXact_UndoLog(bool isCommit, TransactionId xid);
extern void AtPrepare_UndoLog(void);
extern void UndoLog_UndoByXid(bool isCommit, TransactionId xid,
int nchildren, TransactionId *children);
--
2.43.5
----Next_Part(Fri_Dec_27_17_25_02_2024_357)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="v36-0015-Add-test-for-ALTER-TABLE-UNLOGGED.patch"
view thread (4+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH v36 14/17] In-place persistance change to UNLOGGED
In-Reply-To: <no-message-id-57781@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox