From: Kyotaro Horiguchi Date: Tue, 27 Aug 2024 11:19:53 +0900 Subject: [PATCH v36 14/17] In-place persistance change to UNLOGGED This commit enables changing the persistence of relations to UNLOGGED without creating a new storage file. ALTER TABLE LOGGED will continue to create a new storage as before. --- src/backend/access/transam/undolog.c | 32 +++- src/backend/access/transam/xact.c | 8 +- src/backend/catalog/storage.c | 112 ++++++++++++- src/backend/commands/tablecmds.c | 226 ++++++++++++++++++++++----- src/include/access/undolog.h | 2 +- 5 files changed, 331 insertions(+), 49 deletions(-) diff --git a/src/backend/access/transam/undolog.c b/src/backend/access/transam/undolog.c index b2fdbfcd0f9..33d1b35ae8d 100644 --- a/src/backend/access/transam/undolog.c +++ b/src/backend/access/transam/undolog.c @@ -757,9 +757,10 @@ UndoLog_UndoByXid(bool isCommit, TransactionId xid, * During recovery, it should pass the target transaction ID. */ void -AtEOXact_UndoLog(TransactionId xid) +AtEOXact_UndoLog(bool isCommit, TransactionId xid) { FullTransactionId fxid = ULogLocal.current_xid; + bool redo = false; if (TransactionIdIsValid(xid)) { @@ -767,7 +768,8 @@ AtEOXact_UndoLog(TransactionId xid) TransactionId oldest_xid; TransactionId next_xid; uint32 oldest_epoch; - + + redo = true; LWLockAcquire(XactTruncationLock, LW_SHARED); next_fxid = TransamVariables->nextXid; oldest_xid = TransamVariables->oldestClogXid; @@ -785,7 +787,31 @@ AtEOXact_UndoLog(TransactionId xid) } if (FullTransactionIdIsValid(fxid)) - undolog_drop_ulog(fxid); + { + UndoLogSlot *slot; + + slot = undolog_find_slot(fxid, false); + if (slot) + { + undolog_flush_slot(slot, false); + LWLockRelease(&slot->lock); + } + + if (slot || undolog_file_exists(fxid)) + { + char fname[MAXPGPATH]; + ULogContext cxt; + + if (isCommit) + cxt = ULOGCXT_COMMIT; + else + cxt = ULOGCXT_ABORT; + + UndoLogSetFilename(fname, fxid); + undolog_process_ulog(fname, cxt, redo); + undolog_drop_ulog(fxid); + } + } } /* diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index b609a783464..197fde27edc 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -2480,7 +2480,7 @@ CommitTransaction(void) AtEOXact_on_commit_actions(true); AtEOXact_Namespace(true, is_parallel_worker); AtEOXact_SMgr(); - AtEOXact_UndoLog(InvalidTransactionId); + AtEOXact_UndoLog(true, InvalidTransactionId); AtEOXact_Files(true); AtEOXact_ComboCid(); AtEOXact_HashTables(true); @@ -2999,7 +2999,7 @@ AbortTransaction(void) AtEOXact_on_commit_actions(false); AtEOXact_Namespace(false, is_parallel_worker); AtEOXact_SMgr(); - AtEOXact_UndoLog(InvalidTransactionId); + AtEOXact_UndoLog(false, InvalidTransactionId); AtEOXact_Files(false); AtEOXact_ComboCid(); AtEOXact_HashTables(false); @@ -6269,7 +6269,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed, true); } - AtEOXact_UndoLog(xid); + AtEOXact_UndoLog(true, xid); AtEOXact_Buffers_Redo(true, xid, parsed->nsubxacts, parsed->subxacts); if (parsed->nstats > 0) @@ -6384,7 +6384,7 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid, true); } - AtEOXact_UndoLog(xid); + AtEOXact_UndoLog(false, xid); AtEOXact_Buffers_Redo(false, xid, parsed->nsubxacts, parsed->subxacts); if (parsed->nstats > 0) diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index 67f2b3727a9..ad8855b69b4 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -32,6 +32,8 @@ #include "miscadmin.h" #include "storage/bulk_write.h" #include "storage/freespace.h" +#include "storage/copydir.h" +#include "storage/fd.h" #include "storage/proc.h" #include "storage/smgr.h" #include "utils/hsearch.h" @@ -558,6 +560,58 @@ RelationTruncate(Relation rel, BlockNumber nblocks) FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber); } +/* + * Reset an unlogged relation using the INIT fork, intended for use during the + * commit of prepared transactions. The relation is assumed to be UNLOGGED, so + * no WAL-logging is required. + */ +static void +ResetUnloggedRelation(RelFileLocator rloc, ProcNumber backend) +{ + char *srcpath; + char *dstpath; + SMgrRelation srel = smgropen(rloc, backend); + ForkNumber forks[MAX_FORKNUM]; + BlockNumber blocks[MAX_FORKNUM]; + BlockNumber old_blocks[MAX_FORKNUM]; + int nforks = 0; + + srel = smgropen(rloc, backend); + + Assert(smgrexists(srel, INIT_FORKNUM)); + + for (int i = 0 ; i <= MAX_FORKNUM ; i++) + { + if (i == INIT_FORKNUM || !smgrexists(srel, i)) + continue; + + forks[nforks] = i; + old_blocks[nforks] = smgrnblocks(srel, i); + blocks[nforks] = 0; + nforks++; + } + + /* + * This relation is unlogged. Therefore, unlike RelationTruncate(), there + * is no need to call RelationPreTruncate(). + */ + START_CRIT_SECTION(); + smgrtruncate(srel, forks, nforks, old_blocks, blocks); + END_CRIT_SECTION(); + + /* Note that this leaves the first segment of the main fork. */ + for (int i = 0 ; i < nforks ; i++) + smgrunlink(srel, forks[i], false); + + /* copy init fork to main fork */ + srcpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, INIT_FORKNUM); + dstpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, MAIN_FORKNUM); + copy_file_extended(srcpath, dstpath, true); + fsync_fname(dstpath, false); +} + /* * RelationPreTruncate * Perform AM-independent work before a physical truncation. @@ -1314,8 +1368,62 @@ smgr_undo(UndoLogRecord *record, ULogContext cxt, bool redo, bool crashed) else elog(PANIC, "smgr_undo: unknown op code %d", info); } - else if (cxt == ULOGCXT_COMMIT || cxt == ULOGCXT_ABORT || - cxt == ULOGCXT_PREPARED) + else if (cxt == ULOGCXT_COMMIT) + { + Assert(record); + info = record->ul_info & ~ULR_INFO_MASK; + + if (info == ULOG_SMGR_CREATE) + { + ul_smgr_create *ulrec = (ul_smgr_create *) ULogRecGetData(record); + /* + * If an init fork was created during recovery, the entire relation + * is set to be reset at recovery-end or the consistency point. + * Therefore, we need to drop the relation's buffers to prevent the + * end-of-recovery checkpoint from flushing storage files for these + * relations once they have been reset. + */ + if (redo && ulrec->forknum == INIT_FORKNUM) + { + SMgrRelation reln; + int nforks; + ForkNumber forks[MAX_FORKNUM + 1]; + BlockNumber firstblocks[MAX_FORKNUM + 1] = {0}; + + Assert(ulrec->backend == INVALID_PROC_NUMBER); + + reln = smgropen(ulrec->rlocator, ulrec->backend); + + nforks = 0; + for (int i = 0 ; i <= MAX_FORKNUM ; i++) + { + if (smgrexists(reln, i)) + forks[nforks++] = i; + } + + if (nforks > 0) + DropRelationBuffers(reln, forks, nforks, firstblocks); + + smgrclose(reln); + } + else if (!redo && crashed && ulrec->forknum == INIT_FORKNUM) + { + /* + * System has been crashed until the transaction was + * prepared. Now that the init fork is persists, the relation + * needs to be cleared. + */ + ResetUnloggedRelation(ulrec->rlocator, ulrec->backend); + ereport(WARNING, + errmsg("unlogged relation %u/%u/%u was reset", + ulrec->rlocator.spcOid, ulrec->rlocator.dbOid, + ulrec->rlocator.relNumber), + errdetail("Server experinced a crash after the transaction that altered the relation was prepared.")); + } + + } + } + else if(cxt == ULOGCXT_PREPARED || cxt == ULOGCXT_ABORT) { /* nothing to do here */ } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b5766989d8e..3c58d5be464 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5744,6 +5744,143 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, return newcmd; } +/* + * RelationChangePersistence: perform in-place persistence change of a relation + */ +static void +RelationChangePersistence(AlteredTableInfo *tab, char persistence, + LOCKMODE lockmode) +{ + Relation rel; + Relation classRel; + HeapTuple tuple, + newtuple; + Datum new_val[Natts_pg_class]; + bool new_null[Natts_pg_class], + new_repl[Natts_pg_class]; + List *relids; + ListCell *lc_oid; + + Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE); + Assert(lockmode == AccessExclusiveLock); + + /* + * Use ATRewriteTable instead of this function if the following condition + * is not satisfied. + */ + Assert(tab->constraints == NULL && tab->partition_constraint == NULL && + tab->newvals == NULL && !tab->verify_new_notnull); + + rel = table_open(tab->relid, lockmode); + + Assert(rel->rd_rel->relpersistence != persistence); + + elog(DEBUG1, "perform in-place persistence change"); + + /* + * Initially, gather all relations that require a persistence change. + */ + + /* Collect OIDs of indexes and toast relations */ + relids = RelationGetIndexList(rel); + relids = lcons_oid(rel->rd_id, relids); + + /* Add toast relation if any */ + if (OidIsValid(rel->rd_rel->reltoastrelid)) + { + List *toastidx; + Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode); + + relids = lappend_oid(relids, rel->rd_rel->reltoastrelid); + toastidx = RelationGetIndexList(toastrel); + relids = list_concat(relids, toastidx); + pfree(toastidx); + table_close(toastrel, NoLock); + } + + table_close(rel, NoLock); + + /* Make changes in storage */ + classRel = table_open(RelationRelationId, RowExclusiveLock); + + foreach(lc_oid, relids) + { + Oid reloid = lfirst_oid(lc_oid); + Relation r = relation_open(reloid, lockmode); + SMgrRelation srel; + bool persistent = (persistence == RELPERSISTENCE_PERMANENT); + bool is_index; + + /* + * Reconstruct the storage when permanent and unlogged storage types + * are incompatible. + */ + if (r->rd_rel->relkind == RELKIND_INDEX && + !r->rd_indam->amunloggedstoragecompatible) + { + int reindex_flags; + ReindexParams params = {0}; + + /* reindex doesn't allow concurrent use of the index */ + table_close(r, NoLock); + + reindex_flags = + REINDEX_REL_SUPPRESS_INDEX_USE | + REINDEX_REL_CHECK_CONSTRAINTS; + + /* Set the same persistence with the parent relation. */ + if (persistent) + reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; + else + reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED; + + /* this doesn't fire REINDEX event triegger */ + reindex_index(NULL, reloid, reindex_flags, persistence, ¶ms); + + continue; + } + + /* Currently, only allowing changes to UNLOGGED. */ + Assert(!persistent); + + RelationAssumePersistenceChange(r); + + /* switch buffer persistence */ + srel = RelationGetSmgr(r); + log_smgrbufpersistence(srel->smgr_rlocator.locator, persistent); + SetRelationBuffersPersistence(srel, persistent); + + /* then create the init fork */ + is_index = (r->rd_rel->relkind == RELKIND_INDEX); + RelationCreateFork(srel, INIT_FORKNUM, !is_index, true); + if (is_index) + r->rd_indam->ambuildempty(r); + + /* Update catalog */ + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", reloid); + + memset(new_val, 0, sizeof(new_val)); + memset(new_null, false, sizeof(new_null)); + memset(new_repl, false, sizeof(new_repl)); + + new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence); + new_null[Anum_pg_class_relpersistence - 1] = false; + new_repl[Anum_pg_class_relpersistence - 1] = true; + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel), + new_val, new_null, new_repl); + + CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple); + heap_freetuple(newtuple); + + table_close(r, NoLock); + } + + table_close(classRel, NoLock); +} + /* * ATRewriteTables: ALTER TABLE phase 3 */ @@ -5876,48 +6013,59 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode, tab->relid, tab->rewrite); - /* - * Create transient table that will receive the modified data. - * - * Ensure it is marked correctly as logged or unlogged. We have - * to do this here so that buffers for the new relfilenumber will - * have the right persistence set, and at the same time ensure - * that the original filenumbers's buffers will get read in with - * the correct setting (i.e. the original one). Otherwise a - * rollback after the rewrite would possibly result with buffers - * for the original filenumbers having the wrong persistence - * setting. - * - * NB: This relies on swap_relation_files() also swapping the - * persistence. That wouldn't work for pg_class, but that can't be - * unlogged anyway. - */ - OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod, - persistence, lockmode); + if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE && + persistence == RELPERSISTENCE_UNLOGGED) + { + /* Make in-place persistence change. */ + RelationChangePersistence(tab, persistence, lockmode); + } + else + { + /* + * Create transient table that will receive the modified data. + * + * Ensure it is marked correctly as logged or unlogged. We + * have to do this here so that buffers for the new + * relfilenumber will have the right persistence set, and at + * the same time ensure that the original filenumbers's buffers + * will get read in with the correct setting (i.e. the original + * one). Otherwise a rollback after the rewrite would possibly + * result with buffers for the original filenumbers having the + * wrong persistence setting. + * + * NB: This relies on swap_relation_files() also swapping the + * persistence. That wouldn't work for pg_class, but that can't + * be unlogged anyway. + */ + OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, + NewAccessMethod, + persistence, lockmode); - /* - * Copy the heap data into the new table with the desired - * modifications, and test the current data within the table - * against new constraints generated by ALTER TABLE commands. - */ - ATRewriteTable(tab, OIDNewHeap, lockmode); + /* + * Copy the heap data into the new table with the desired + * modifications, and test the current data within the table + * against new constraints generated by ALTER TABLE commands. + */ + ATRewriteTable(tab, OIDNewHeap, lockmode); - /* - * Swap the physical files of the old and new heaps, then rebuild - * indexes and discard the old heap. We can use RecentXmin for - * the table's new relfrozenxid because we rewrote all the tuples - * in ATRewriteTable, so no older Xid remains in the table. Also, - * we never try to swap toast tables by content, since we have no - * interest in letting this code work on system catalogs. - */ - finish_heap_swap(tab->relid, OIDNewHeap, - false, false, true, - !OidIsValid(tab->newTableSpace), - RecentXmin, - ReadNextMultiXactId(), - persistence); + /* + * Swap the physical files of the old and new heaps, then + * rebuild indexes and discard the old heap. We can use + * RecentXmin for the table's new relfrozenxid because we + * rewrote all the tuples in ATRewriteTable, so no older Xid + * remains in the table. Also, we never try to swap toast + * tables by content, since we have no interest in letting this + * code work on system catalogs. + */ + finish_heap_swap(tab->relid, OIDNewHeap, + false, false, true, + !OidIsValid(tab->newTableSpace), + RecentXmin, + ReadNextMultiXactId(), + persistence); - InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + } } else if (tab->rewrite > 0 && tab->relkind == RELKIND_SEQUENCE) { diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h index 19badc852a0..857a845eb9d 100644 --- a/src/include/access/undolog.h +++ b/src/include/access/undolog.h @@ -80,7 +80,7 @@ extern Size UndoLogShmemSize(void); extern void UndoLogShmemInit(void); extern void InitUndoLog(void); extern void UndoLogWrite(RmgrId rmgr, uint8 info, void *data, int len); -extern void AtEOXact_UndoLog(TransactionId xid); +extern void AtEOXact_UndoLog(bool isCommit, TransactionId xid); extern void AtPrepare_UndoLog(void); extern void UndoLog_UndoByXid(bool isCommit, TransactionId xid, int nchildren, TransactionId *children); -- 2.43.5 ----Next_Part(Fri_Dec_27_17_25_02_2024_357)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v36-0015-Add-test-for-ALTER-TABLE-UNLOGGED.patch"