agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/8] Alter table set compression 4+ messages / 2 participants [nested] [flat]
* [PATCH 7/8] Alter table set compression @ 2021-03-10 09:04 Dilip Kumar <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Dilip Kumar @ 2021-03-10 09:04 UTC (permalink / raw) Add support for changing the compression method associated with a column. There are only built-in methods so we don't need to rewrite the table, only the new tuples will be compressed with the new compression method. Dilip Kumar based on the patches from Ildus Kurbangaliev. Design input from Robert Haas and Tomas Vondra. Reviewed by Robert Haas, Tomas Vondra, Alexander Korotkov and Justin Pryzby Discussions: https://www.postgresql.org/message-id/[email protected] https://www.postgresql.org/message-id/CA%2BTgmoaKDW1Oi9V%3Djc9hOGyf77NbkNEABuqgHD1Cq%3D%3D1QsOcxg%40... https://www.postgresql.org/message-id/CA%2BTgmobSDVgUage9qQ5P_%3DF_9jaMkCgyKxUQGtFQU7oN4kX-AA%40mail... https://www.postgresql.org/message-id/20201005160355.byp74sh3ejsv7wrj%40development https://www.postgresql.org/message-id/CAFiTN-tzTTT2oqWdRGLv1dvvS5MC1W%2BLE%2B3bqWPJUZj4GnHOJg%40mail... --- doc/src/sgml/ref/alter_table.sgml | 16 ++ src/backend/commands/tablecmds.c | 198 +++++++++++++++----- src/backend/parser/gram.y | 9 + src/bin/psql/tab-complete.c | 2 +- src/include/nodes/parsenodes.h | 3 +- src/test/regress/expected/compression.out | 47 ++++- src/test/regress/expected/compression_1.out | 48 ++++- src/test/regress/sql/compression.sql | 20 ++ 8 files changed, 296 insertions(+), 47 deletions(-) diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 38e416d183..e147831640 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -54,6 +54,7 @@ ALTER TABLE [ IF EXISTS ] <replaceable class="parameter">name</replaceable> ALTER [ COLUMN ] <replaceable class="parameter">column_name</replaceable> SET ( <replaceable class="parameter">attribute_option</replaceable> = <replaceable class="parameter">value</replaceable> [, ... ] ) ALTER [ COLUMN ] <replaceable class="parameter">column_name</replaceable> RESET ( <replaceable class="parameter">attribute_option</replaceable> [, ... ] ) ALTER [ COLUMN ] <replaceable class="parameter">column_name</replaceable> SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER [ COLUMN ] <replaceable class="parameter">column_name</replaceable> SET COMPRESSION <replaceable class="parameter">compression_method</replaceable> ADD <replaceable class="parameter">table_constraint</replaceable> [ NOT VALID ] ADD <replaceable class="parameter">table_constraint_using_index</replaceable> ALTER CONSTRAINT <replaceable class="parameter">constraint_name</replaceable> [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -104,6 +105,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( <replaceable>sequence_options</replaceable> ) ] | UNIQUE <replaceable class="parameter">index_parameters</replaceable> | PRIMARY KEY <replaceable class="parameter">index_parameters</replaceable> | + COMPRESSION <replaceable class="parameter">compression_method</replaceable> | REFERENCES <replaceable class="parameter">reftable</replaceable> [ ( <replaceable class="parameter">refcolumn</replaceable> ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE <replaceable class="parameter">referential_action</replaceable> ] [ ON UPDATE <replaceable class="parameter">referential_action</replaceable> ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -384,6 +386,20 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </listitem> </varlistentry> + <varlistentry> + <term> + <literal>SET COMPRESSION <replaceable class="parameter">compression_method</replaceable></literal> + </term> + <listitem> + <para> + This sets the compression method for a column. The supported compression + methods are <literal>pglz</literal> and <literal>lz4</literal>. + <literal>lz4</literal> is available only if <literal>--with-lz4</literal> + was used when building <productname>PostgreSQL</productname>. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><literal>ADD <replaceable class="parameter">table_constraint</replaceable> [ NOT VALID ]</literal></term> <listitem> diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a0d2af0cde..bb284741ce 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -532,6 +532,8 @@ static void ATExecReplicaIdentity(Relation rel, ReplicaIdentityStmt *stmt, LOCKM static void ATExecGenericOptions(Relation rel, List *options); static void ATExecSetRowSecurity(Relation rel, bool rls); static void ATExecForceNoForceRowSecurity(Relation rel, bool force_rls); +static ObjectAddress ATExecSetCompression(AlteredTableInfo *tab, Relation rel, + const char *column, Node *newValue, LOCKMODE lockmode); static void index_copy_data(Relation rel, RelFileNode newrnode); static const char *storage_name(char c); @@ -3977,6 +3979,7 @@ AlterTableGetLockLevel(List *cmds) */ case AT_GenericOptions: case AT_AlterColumnGenericOptions: + case AT_SetCompression: cmd_lockmode = AccessExclusiveLock; break; @@ -4511,7 +4514,8 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: - ATSimplePermissions(rel, ATT_TABLE); + case AT_SetCompression: + ATSimplePermissions(rel, ATT_TABLE | ATT_MATVIEW); /* These commands never recurse */ /* No command-specific prep needed */ pass = AT_PASS_MISC; @@ -4924,6 +4928,10 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, Assert(rel->rd_rel->relkind == RELKIND_INDEX); ATExecAlterCollationRefreshVersion(rel, cmd->object); break; + case AT_SetCompression: + address = ATExecSetCompression(tab, rel, cmd->name, cmd->def, + lockmode); + break; default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -7812,6 +7820,67 @@ ATExecSetOptions(Relation rel, const char *colName, Node *options, return address; } +/* + * Helper function for ATExecSetStorage and ATExecSetCompression + * + * Set the attcompression and/or attstorage for the respective index attribute + * if the respective input values are valid. + */ +static void +ApplyChangesToIndexes(Relation rel, Relation attrelation, AttrNumber attnum, + char newcompression, char newstorage, LOCKMODE lockmode) +{ + HeapTuple tuple; + ListCell *lc; + Form_pg_attribute attrtuple; + + foreach(lc, RelationGetIndexList(rel)) + { + Oid indexoid = lfirst_oid(lc); + Relation indrel; + AttrNumber indattnum = 0; + + indrel = index_open(indexoid, lockmode); + + for (int i = 0; i < indrel->rd_index->indnatts; i++) + { + if (indrel->rd_index->indkey.values[i] == attnum) + { + indattnum = i + 1; + break; + } + } + + if (indattnum == 0) + { + index_close(indrel, lockmode); + continue; + } + + tuple = SearchSysCacheCopyAttNum(RelationGetRelid(indrel), indattnum); + + if (HeapTupleIsValid(tuple)) + { + attrtuple = (Form_pg_attribute) GETSTRUCT(tuple); + + if (CompressionMethodIsValid(newcompression)) + attrtuple->attcompression = newcompression; + + if (newstorage != '\0') + attrtuple->attstorage = newstorage; + + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + attrtuple->attnum); + + heap_freetuple(tuple); + } + + index_close(indrel, lockmode); + } +} /* * ALTER TABLE ALTER COLUMN SET STORAGE * @@ -7827,7 +7896,6 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc Form_pg_attribute attrtuple; AttrNumber attnum; ObjectAddress address; - ListCell *lc; Assert(IsA(newValue, String)); storagemode = strVal(newValue); @@ -7891,47 +7959,8 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc * Apply the change to indexes as well (only for simple index columns, * matching behavior of index.c ConstructTupleDescriptor()). */ - foreach(lc, RelationGetIndexList(rel)) - { - Oid indexoid = lfirst_oid(lc); - Relation indrel; - AttrNumber indattnum = 0; - - indrel = index_open(indexoid, lockmode); - - for (int i = 0; i < indrel->rd_index->indnatts; i++) - { - if (indrel->rd_index->indkey.values[i] == attnum) - { - indattnum = i + 1; - break; - } - } - - if (indattnum == 0) - { - index_close(indrel, lockmode); - continue; - } - - tuple = SearchSysCacheCopyAttNum(RelationGetRelid(indrel), indattnum); - - if (HeapTupleIsValid(tuple)) - { - attrtuple = (Form_pg_attribute) GETSTRUCT(tuple); - attrtuple->attstorage = newstorage; - - CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); - - InvokeObjectPostAlterHook(RelationRelationId, - RelationGetRelid(rel), - attrtuple->attnum); - - heap_freetuple(tuple); - } - - index_close(indrel, lockmode); - } + ApplyChangesToIndexes(rel, attrelation, attnum, InvalidCompressionMethod, + newstorage, lockmode); table_close(attrelation, RowExclusiveLock); @@ -15118,6 +15147,89 @@ ATExecGenericOptions(Relation rel, List *options) heap_freetuple(tuple); } +/* + * ALTER TABLE ALTER COLUMN SET COMPRESSION + * + * Return value is the address of the modified column + */ +static ObjectAddress +ATExecSetCompression(AlteredTableInfo *tab, + Relation rel, + const char *column, + Node *newValue, + LOCKMODE lockmode) +{ + Relation attrel; + HeapTuple tuple; + Form_pg_attribute atttableform; + AttrNumber attnum; + char *compression; + char typstorage; + Oid cmoid; + Datum values[Natts_pg_attribute]; + bool nulls[Natts_pg_attribute]; + bool replace[Natts_pg_attribute]; + ObjectAddress address; + + Assert(IsA(newValue, String)); + compression = strVal(newValue); + + attrel = table_open(AttributeRelationId, RowExclusiveLock); + + tuple = SearchSysCacheAttName(RelationGetRelid(rel), column); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + column, RelationGetRelationName(rel)))); + + /* prevent them from altering a system attribute */ + atttableform = (Form_pg_attribute) GETSTRUCT(tuple); + attnum = atttableform->attnum; + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", column))); + + typstorage = get_typstorage(atttableform->atttypid); + + /* prevent from setting compression methods for uncompressible type */ + if (!IsStorageCompressible(typstorage)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column data type %s does not support compression", + format_type_be(atttableform->atttypid)))); + + /* initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replace, false, sizeof(replace)); + + /* get the attribute compression method. */ + cmoid = GetAttributeCompression(atttableform, compression); + + atttableform->attcompression = cmoid; + CatalogTupleUpdate(attrel, &tuple->t_self, tuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + atttableform->attnum); + + ReleaseSysCache(tuple); + + /* apply changes to the index column as well */ + ApplyChangesToIndexes(rel, attrel, attnum, cmoid, '\0', lockmode); + table_close(attrel, RowExclusiveLock); + + /* make changes visible */ + CommandCounterIncrement(); + + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), atttableform->attnum); + return address; +} + + /* * Preparation phase for SET LOGGED/UNLOGGED * diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index f8bc11d28b..07aa3c9330 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -2308,6 +2308,15 @@ alter_table_cmd: n->missing_ok = true; $$ = (Node *)n; } + /* ALTER TABLE <name> ALTER [COLUMN] <colname> SET (COMPRESSION <cm>) */ + | ALTER opt_column ColId SET optColumnCompression + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_SetCompression; + n->name = $3; + n->def = (Node *) makeString($5); + $$ = (Node *)n; + } /* ALTER TABLE <name> DROP [COLUMN] IF EXISTS <colname> [RESTRICT|CASCADE] */ | DROP opt_column IF_P EXISTS ColId opt_drop_behavior { diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index fdd1bdd92a..ebe07a17cd 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2115,7 +2115,7 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] <foo> SET */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET")) - COMPLETE_WITH("(", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); + COMPLETE_WITH("(", "COMPRESSION", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); /* ALTER TABLE ALTER [COLUMN] <foo> SET ( */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "(")) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 56e24529f1..b20c03f691 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1902,7 +1902,8 @@ typedef enum AlterTableType AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ AT_DropIdentity, /* DROP IDENTITY */ - AT_AlterCollationRefreshVersion /* ALTER COLLATION ... REFRESH VERSION */ + AT_AlterCollationRefreshVersion, /* ALTER COLLATION ... REFRESH VERSION */ + AT_SetCompression /* SET COMPRESSION */ } AlterTableType; typedef struct ReplicaIdentityStmt diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out index 9ddffc8b18..19625708a9 100644 --- a/src/test/regress/expected/compression.out +++ b/src/test/regress/expected/compression.out @@ -247,12 +247,57 @@ CREATE TABLE cmdata2 (f1 text); --------+------+-----------+----------+---------+----------+-------------+--------------+------------- f1 | text | | | | extended | lz4 | | +-- test alter compression method +ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; +INSERT INTO cmdata VALUES (repeat('123456789',4004)); +\d+ cmdata + Table "public.cmdata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | lz4 | | +Indexes: + "idx" btree (f1) + +SELECT pg_column_compression(f1) FROM cmdata; + pg_column_compression +----------------------- + pglz + lz4 +(2 rows) + +-- test alter compression method for the materialized view +ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; +\d+ mv + Materialized view "public.mv" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + x | text | | | | extended | lz4 | | +View definition: + SELECT cmdata1.f1 AS x + FROM cmdata1; + +-- test alter compression method for the partitioned table +ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; +ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; +-- new data should be compressed with the current compression method +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +INSERT INTO cmpart VALUES (repeat('123456789',4004)); +SELECT pg_column_compression(f1) FROM cmpart; + pg_column_compression +----------------------- + lz4 + pglz + pglz + lz4 +(4 rows) + -- check data is ok SELECT length(f1) FROM cmdata; length -------- 10000 -(1 row) + 36036 +(2 rows) SELECT length(f1) FROM cmdata1; length diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out index 03a7f46554..bd642634a6 100644 --- a/src/test/regress/expected/compression_1.out +++ b/src/test/regress/expected/compression_1.out @@ -256,12 +256,58 @@ CREATE TABLE cmdata2 (f1 text); --------+------+-----------+----------+---------+----------+-------------+--------------+------------- f1 | text | | | | extended | pglz | | +-- test alter compression method +ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; +ERROR: unsupported LZ4 compression method +DETAIL: This functionality requires the server to be built with lz4 support. +HINT: You need to rebuild PostgreSQL using --with-lz4. +INSERT INTO cmdata VALUES (repeat('123456789',4004)); +\d+ cmdata + Table "public.cmdata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | +Indexes: + "idx" btree (f1) + +SELECT pg_column_compression(f1) FROM cmdata; + pg_column_compression +----------------------- + pglz + pglz +(2 rows) + +-- test alter compression method for the materialized view +ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; +ERROR: relation "mv" does not exist +\d+ mv +-- test alter compression method for the partitioned table +ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; +ERROR: relation "cmpart1" does not exist +ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; +ERROR: unsupported LZ4 compression method +DETAIL: This functionality requires the server to be built with lz4 support. +HINT: You need to rebuild PostgreSQL using --with-lz4. +-- new data should be compressed with the current compression method +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +ERROR: relation "cmpart" does not exist +LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',1004)); + ^ +INSERT INTO cmpart VALUES (repeat('123456789',4004)); +ERROR: relation "cmpart" does not exist +LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',4004)); + ^ +SELECT pg_column_compression(f1) FROM cmpart; +ERROR: relation "cmpart" does not exist +LINE 1: SELECT pg_column_compression(f1) FROM cmpart; + ^ -- check data is ok SELECT length(f1) FROM cmdata; length -------- 10000 -(1 row) + 36036 +(2 rows) SELECT length(f1) FROM cmdata1; ERROR: relation "cmdata1" does not exist diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql index 6c9b24f1f7..00b1ff1137 100644 --- a/src/test/regress/sql/compression.sql +++ b/src/test/regress/sql/compression.sql @@ -111,6 +111,26 @@ DROP TABLE cmdata2; CREATE TABLE cmdata2 (f1 text); \d+ cmdata2 +-- test alter compression method +ALTER TABLE cmdata ALTER COLUMN f1 SET COMPRESSION lz4; +INSERT INTO cmdata VALUES (repeat('123456789',4004)); +\d+ cmdata +SELECT pg_column_compression(f1) FROM cmdata; + +-- test alter compression method for the materialized view +ALTER MATERIALIZED VIEW mv ALTER COLUMN x SET COMPRESSION lz4; +\d+ mv + +-- test alter compression method for the partitioned table +ALTER TABLE cmpart1 ALTER COLUMN f1 SET COMPRESSION pglz; +ALTER TABLE cmpart2 ALTER COLUMN f1 SET COMPRESSION lz4; + +-- new data should be compressed with the current compression method +INSERT INTO cmpart VALUES (repeat('123456789',1004)); +INSERT INTO cmpart VALUES (repeat('123456789',4004)); + +SELECT pg_column_compression(f1) FROM cmpart; + -- check data is ok SELECT length(f1) FROM cmdata; SELECT length(f1) FROM cmdata1; -- 2.17.0 --C94crkcyjafcjHxo Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0008-default-to-with-lz4.patch" ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v35 15/21] In-place persistance change to UNLOGGED @ 2024-08-27 02:19 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Kyotaro Horiguchi @ 2024-08-27 02:19 UTC (permalink / raw) This commit enables changing the persistence of relations to UNLOGGED without creating a new storage file. ALTER TABLE LOGGED will continue to create a new storage as before. --- src/backend/catalog/storage.c | 64 +++++++++ src/backend/commands/tablecmds.c | 226 +++++++++++++++++++++++++------ 2 files changed, 251 insertions(+), 39 deletions(-) diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index 51be987a5f8..4a2c620402b 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -458,6 +458,54 @@ RelationTruncate(Relation rel, BlockNumber nblocks) FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber); } +/* + * Reset an unlogged relation using the INIT fork, intended for use during the + * commit of prepared transactions. The relation is assumed to be UNLOGGED, so + * no WAL-logging is required. + */ +static void +ResetUnloggedRelation(RelFileLocator rloc, ProcNumber backend) +{ + char *srcpath; + char *dstpath; + SMgrRelation srel = smgropen(rloc, backend); + ForkNumber forks[MAX_FORKNUM]; + BlockNumber blocks[MAX_FORKNUM]; + int nforks = 0; + + srel = smgropen(rloc, backend); + + Assert(smgrexists(srel, INIT_FORKNUM)); + + for (int i = 0 ; i <= MAX_FORKNUM ; i++) + { + if (i == INIT_FORKNUM || !smgrexists(srel, i)) + continue; + + forks[nforks] = i; + blocks[nforks] = 0; + nforks++; + } + + /* + * This relation is unlogged. Therefore, unlike RelationTruncate(), there + * is no need to call RelationPreTruncate(). + */ + smgrtruncate(srel, forks, nforks, blocks); + + /* Note that this leaves the first segment of the main fork. */ + for (int i = 0 ; i < nforks ; i++) + smgrunlink(srel, forks[i], false); + + /* copy init fork to main fork */ + srcpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, INIT_FORKNUM); + dstpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, MAIN_FORKNUM); + copy_file_extended(srcpath, dstpath, true); + fsync_fname(dstpath, false); +} + /* * RelationPreTruncate * Perform AM-independent work before a physical truncation. @@ -1181,6 +1229,22 @@ smgr_undo(UndoLogRecord *record, ULogOp op, bool recovered, bool redo) smgrclose(reln); } + else if (!redo && recovered && ulrec->forknum == INIT_FORKNUM) + { + /* + * System has been crashed until the transaction was + * prepared. Now that the init fork is persists, the relation + * needs to be cleared. + */ + ResetUnloggedRelation(ulrec->rlocator, ulrec->backend); + ereport(WARNING, + errmsg("unlogged relation %u/%u/%u was reset", + ulrec->rlocator.spcOid, ulrec->rlocator.dbOid, + ulrec->rlocator.relNumber), + errdetail("Server experinced a crash after the transaction that altered the relation was prepared.")); + + } + } else elog(PANIC, "smgr_undo: unknown ulogop code %d", op); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 251aea55d24..f8d240f374f 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5708,6 +5708,143 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, return newcmd; } +/* + * RelationChangePersistence: perform in-place persistence change of a relation + */ +static void +RelationChangePersistence(AlteredTableInfo *tab, char persistence, + LOCKMODE lockmode) +{ + Relation rel; + Relation classRel; + HeapTuple tuple, + newtuple; + Datum new_val[Natts_pg_class]; + bool new_null[Natts_pg_class], + new_repl[Natts_pg_class]; + List *relids; + ListCell *lc_oid; + + Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE); + Assert(lockmode == AccessExclusiveLock); + + /* + * Use ATRewriteTable instead of this function if the following condition + * is not satisfied. + */ + Assert(tab->constraints == NULL && tab->partition_constraint == NULL && + tab->newvals == NULL && !tab->verify_new_notnull); + + rel = table_open(tab->relid, lockmode); + + Assert(rel->rd_rel->relpersistence != persistence); + + elog(DEBUG1, "perform in-place persistence change"); + + /* + * Initially, gather all relations that require a persistence change. + */ + + /* Collect OIDs of indexes and toast relations */ + relids = RelationGetIndexList(rel); + relids = lcons_oid(rel->rd_id, relids); + + /* Add toast relation if any */ + if (OidIsValid(rel->rd_rel->reltoastrelid)) + { + List *toastidx; + Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode); + + relids = lappend_oid(relids, rel->rd_rel->reltoastrelid); + toastidx = RelationGetIndexList(toastrel); + relids = list_concat(relids, toastidx); + pfree(toastidx); + table_close(toastrel, NoLock); + } + + table_close(rel, NoLock); + + /* Make changes in storage */ + classRel = table_open(RelationRelationId, RowExclusiveLock); + + foreach(lc_oid, relids) + { + Oid reloid = lfirst_oid(lc_oid); + Relation r = relation_open(reloid, lockmode); + SMgrRelation srel; + bool persistent = (persistence == RELPERSISTENCE_PERMANENT); + bool is_index; + + /* + * Reconstruct the storage when permanent and unlogged storage types + * are incompatible. + */ + if (r->rd_rel->relkind == RELKIND_INDEX && + !r->rd_indam->amunloggedstoragecompatible) + { + int reindex_flags; + ReindexParams params = {0}; + + /* reindex doesn't allow concurrent use of the index */ + table_close(r, NoLock); + + reindex_flags = + REINDEX_REL_SUPPRESS_INDEX_USE | + REINDEX_REL_CHECK_CONSTRAINTS; + + /* Set the same persistence with the parent relation. */ + if (persistent) + reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; + else + reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED; + + /* this doesn't fire REINDEX event triegger */ + reindex_index(NULL, reloid, reindex_flags, persistence, ¶ms); + + continue; + } + + /* Currently, only allowing changes to UNLOGGED. */ + Assert(!persistent); + + RelationAssumePersistenceChange(r); + + /* switch buffer persistence */ + srel = RelationGetSmgr(r); + log_smgrbufpersistence(srel->smgr_rlocator.locator, persistent); + SetRelationBuffersPersistence(srel, persistent); + + /* then create the init fork */ + is_index = (r->rd_rel->relkind == RELKIND_INDEX); + RelationCreateFork(srel, INIT_FORKNUM, !is_index, true); + if (is_index) + r->rd_indam->ambuildempty(r); + + /* Update catalog */ + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", reloid); + + memset(new_val, 0, sizeof(new_val)); + memset(new_null, false, sizeof(new_null)); + memset(new_repl, false, sizeof(new_repl)); + + new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence); + new_null[Anum_pg_class_relpersistence - 1] = false; + new_repl[Anum_pg_class_relpersistence - 1] = true; + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel), + new_val, new_null, new_repl); + + CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple); + heap_freetuple(newtuple); + + table_close(r, NoLock); + } + + table_close(classRel, NoLock); +} + /* * ATRewriteTables: ALTER TABLE phase 3 */ @@ -5840,48 +5977,59 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode, tab->relid, tab->rewrite); - /* - * Create transient table that will receive the modified data. - * - * Ensure it is marked correctly as logged or unlogged. We have - * to do this here so that buffers for the new relfilenumber will - * have the right persistence set, and at the same time ensure - * that the original filenumbers's buffers will get read in with - * the correct setting (i.e. the original one). Otherwise a - * rollback after the rewrite would possibly result with buffers - * for the original filenumbers having the wrong persistence - * setting. - * - * NB: This relies on swap_relation_files() also swapping the - * persistence. That wouldn't work for pg_class, but that can't be - * unlogged anyway. - */ - OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod, - persistence, lockmode); + if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE && + persistence == RELPERSISTENCE_UNLOGGED) + { + /* Make in-place persistence change. */ + RelationChangePersistence(tab, persistence, lockmode); + } + else + { + /* + * Create transient table that will receive the modified data. + * + * Ensure it is marked correctly as logged or unlogged. We + * have to do this here so that buffers for the new + * relfilenumber will have the right persistence set, and at + * the same time ensure that the original filenumbers's buffers + * will get read in with the correct setting (i.e. the original + * one). Otherwise a rollback after the rewrite would possibly + * result with buffers for the original filenumbers having the + * wrong persistence setting. + * + * NB: This relies on swap_relation_files() also swapping the + * persistence. That wouldn't work for pg_class, but that can't + * be unlogged anyway. + */ + OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, + NewAccessMethod, + persistence, lockmode); - /* - * Copy the heap data into the new table with the desired - * modifications, and test the current data within the table - * against new constraints generated by ALTER TABLE commands. - */ - ATRewriteTable(tab, OIDNewHeap, lockmode); + /* + * Copy the heap data into the new table with the desired + * modifications, and test the current data within the table + * against new constraints generated by ALTER TABLE commands. + */ + ATRewriteTable(tab, OIDNewHeap, lockmode); - /* - * Swap the physical files of the old and new heaps, then rebuild - * indexes and discard the old heap. We can use RecentXmin for - * the table's new relfrozenxid because we rewrote all the tuples - * in ATRewriteTable, so no older Xid remains in the table. Also, - * we never try to swap toast tables by content, since we have no - * interest in letting this code work on system catalogs. - */ - finish_heap_swap(tab->relid, OIDNewHeap, - false, false, true, - !OidIsValid(tab->newTableSpace), - RecentXmin, - ReadNextMultiXactId(), - persistence); + /* + * Swap the physical files of the old and new heaps, then + * rebuild indexes and discard the old heap. We can use + * RecentXmin for the table's new relfrozenxid because we + * rewrote all the tuples in ATRewriteTable, so no older Xid + * remains in the table. Also, we never try to swap toast + * tables by content, since we have no interest in letting this + * code work on system catalogs. + */ + finish_heap_swap(tab->relid, OIDNewHeap, + false, false, true, + !OidIsValid(tab->newTableSpace), + RecentXmin, + ReadNextMultiXactId(), + persistence); - InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + } } else if (tab->rewrite > 0 && tab->relkind == RELKIND_SEQUENCE) { -- 2.43.5 ----Next_Part(Thu_Oct_31_17_01_30_2024_620)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v35-0016-Add-test-for-ALTER-TABLE-UNLOGGED.patch" ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v36 14/17] In-place persistance change to UNLOGGED @ 2024-08-27 02:19 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Kyotaro Horiguchi @ 2024-08-27 02:19 UTC (permalink / raw) This commit enables changing the persistence of relations to UNLOGGED without creating a new storage file. ALTER TABLE LOGGED will continue to create a new storage as before. --- src/backend/access/transam/undolog.c | 32 +++- src/backend/access/transam/xact.c | 8 +- src/backend/catalog/storage.c | 112 ++++++++++++- src/backend/commands/tablecmds.c | 226 ++++++++++++++++++++++----- src/include/access/undolog.h | 2 +- 5 files changed, 331 insertions(+), 49 deletions(-) diff --git a/src/backend/access/transam/undolog.c b/src/backend/access/transam/undolog.c index b2fdbfcd0f9..33d1b35ae8d 100644 --- a/src/backend/access/transam/undolog.c +++ b/src/backend/access/transam/undolog.c @@ -757,9 +757,10 @@ UndoLog_UndoByXid(bool isCommit, TransactionId xid, * During recovery, it should pass the target transaction ID. */ void -AtEOXact_UndoLog(TransactionId xid) +AtEOXact_UndoLog(bool isCommit, TransactionId xid) { FullTransactionId fxid = ULogLocal.current_xid; + bool redo = false; if (TransactionIdIsValid(xid)) { @@ -767,7 +768,8 @@ AtEOXact_UndoLog(TransactionId xid) TransactionId oldest_xid; TransactionId next_xid; uint32 oldest_epoch; - + + redo = true; LWLockAcquire(XactTruncationLock, LW_SHARED); next_fxid = TransamVariables->nextXid; oldest_xid = TransamVariables->oldestClogXid; @@ -785,7 +787,31 @@ AtEOXact_UndoLog(TransactionId xid) } if (FullTransactionIdIsValid(fxid)) - undolog_drop_ulog(fxid); + { + UndoLogSlot *slot; + + slot = undolog_find_slot(fxid, false); + if (slot) + { + undolog_flush_slot(slot, false); + LWLockRelease(&slot->lock); + } + + if (slot || undolog_file_exists(fxid)) + { + char fname[MAXPGPATH]; + ULogContext cxt; + + if (isCommit) + cxt = ULOGCXT_COMMIT; + else + cxt = ULOGCXT_ABORT; + + UndoLogSetFilename(fname, fxid); + undolog_process_ulog(fname, cxt, redo); + undolog_drop_ulog(fxid); + } + } } /* diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index b609a783464..197fde27edc 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -2480,7 +2480,7 @@ CommitTransaction(void) AtEOXact_on_commit_actions(true); AtEOXact_Namespace(true, is_parallel_worker); AtEOXact_SMgr(); - AtEOXact_UndoLog(InvalidTransactionId); + AtEOXact_UndoLog(true, InvalidTransactionId); AtEOXact_Files(true); AtEOXact_ComboCid(); AtEOXact_HashTables(true); @@ -2999,7 +2999,7 @@ AbortTransaction(void) AtEOXact_on_commit_actions(false); AtEOXact_Namespace(false, is_parallel_worker); AtEOXact_SMgr(); - AtEOXact_UndoLog(InvalidTransactionId); + AtEOXact_UndoLog(false, InvalidTransactionId); AtEOXact_Files(false); AtEOXact_ComboCid(); AtEOXact_HashTables(false); @@ -6269,7 +6269,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed, true); } - AtEOXact_UndoLog(xid); + AtEOXact_UndoLog(true, xid); AtEOXact_Buffers_Redo(true, xid, parsed->nsubxacts, parsed->subxacts); if (parsed->nstats > 0) @@ -6384,7 +6384,7 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid, true); } - AtEOXact_UndoLog(xid); + AtEOXact_UndoLog(false, xid); AtEOXact_Buffers_Redo(false, xid, parsed->nsubxacts, parsed->subxacts); if (parsed->nstats > 0) diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index 67f2b3727a9..ad8855b69b4 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -32,6 +32,8 @@ #include "miscadmin.h" #include "storage/bulk_write.h" #include "storage/freespace.h" +#include "storage/copydir.h" +#include "storage/fd.h" #include "storage/proc.h" #include "storage/smgr.h" #include "utils/hsearch.h" @@ -558,6 +560,58 @@ RelationTruncate(Relation rel, BlockNumber nblocks) FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber); } +/* + * Reset an unlogged relation using the INIT fork, intended for use during the + * commit of prepared transactions. The relation is assumed to be UNLOGGED, so + * no WAL-logging is required. + */ +static void +ResetUnloggedRelation(RelFileLocator rloc, ProcNumber backend) +{ + char *srcpath; + char *dstpath; + SMgrRelation srel = smgropen(rloc, backend); + ForkNumber forks[MAX_FORKNUM]; + BlockNumber blocks[MAX_FORKNUM]; + BlockNumber old_blocks[MAX_FORKNUM]; + int nforks = 0; + + srel = smgropen(rloc, backend); + + Assert(smgrexists(srel, INIT_FORKNUM)); + + for (int i = 0 ; i <= MAX_FORKNUM ; i++) + { + if (i == INIT_FORKNUM || !smgrexists(srel, i)) + continue; + + forks[nforks] = i; + old_blocks[nforks] = smgrnblocks(srel, i); + blocks[nforks] = 0; + nforks++; + } + + /* + * This relation is unlogged. Therefore, unlike RelationTruncate(), there + * is no need to call RelationPreTruncate(). + */ + START_CRIT_SECTION(); + smgrtruncate(srel, forks, nforks, old_blocks, blocks); + END_CRIT_SECTION(); + + /* Note that this leaves the first segment of the main fork. */ + for (int i = 0 ; i < nforks ; i++) + smgrunlink(srel, forks[i], false); + + /* copy init fork to main fork */ + srcpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, INIT_FORKNUM); + dstpath = GetRelationPath(rloc.dbOid, rloc.spcOid, rloc.relNumber, + backend, MAIN_FORKNUM); + copy_file_extended(srcpath, dstpath, true); + fsync_fname(dstpath, false); +} + /* * RelationPreTruncate * Perform AM-independent work before a physical truncation. @@ -1314,8 +1368,62 @@ smgr_undo(UndoLogRecord *record, ULogContext cxt, bool redo, bool crashed) else elog(PANIC, "smgr_undo: unknown op code %d", info); } - else if (cxt == ULOGCXT_COMMIT || cxt == ULOGCXT_ABORT || - cxt == ULOGCXT_PREPARED) + else if (cxt == ULOGCXT_COMMIT) + { + Assert(record); + info = record->ul_info & ~ULR_INFO_MASK; + + if (info == ULOG_SMGR_CREATE) + { + ul_smgr_create *ulrec = (ul_smgr_create *) ULogRecGetData(record); + /* + * If an init fork was created during recovery, the entire relation + * is set to be reset at recovery-end or the consistency point. + * Therefore, we need to drop the relation's buffers to prevent the + * end-of-recovery checkpoint from flushing storage files for these + * relations once they have been reset. + */ + if (redo && ulrec->forknum == INIT_FORKNUM) + { + SMgrRelation reln; + int nforks; + ForkNumber forks[MAX_FORKNUM + 1]; + BlockNumber firstblocks[MAX_FORKNUM + 1] = {0}; + + Assert(ulrec->backend == INVALID_PROC_NUMBER); + + reln = smgropen(ulrec->rlocator, ulrec->backend); + + nforks = 0; + for (int i = 0 ; i <= MAX_FORKNUM ; i++) + { + if (smgrexists(reln, i)) + forks[nforks++] = i; + } + + if (nforks > 0) + DropRelationBuffers(reln, forks, nforks, firstblocks); + + smgrclose(reln); + } + else if (!redo && crashed && ulrec->forknum == INIT_FORKNUM) + { + /* + * System has been crashed until the transaction was + * prepared. Now that the init fork is persists, the relation + * needs to be cleared. + */ + ResetUnloggedRelation(ulrec->rlocator, ulrec->backend); + ereport(WARNING, + errmsg("unlogged relation %u/%u/%u was reset", + ulrec->rlocator.spcOid, ulrec->rlocator.dbOid, + ulrec->rlocator.relNumber), + errdetail("Server experinced a crash after the transaction that altered the relation was prepared.")); + } + + } + } + else if(cxt == ULOGCXT_PREPARED || cxt == ULOGCXT_ABORT) { /* nothing to do here */ } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b5766989d8e..3c58d5be464 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5744,6 +5744,143 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, return newcmd; } +/* + * RelationChangePersistence: perform in-place persistence change of a relation + */ +static void +RelationChangePersistence(AlteredTableInfo *tab, char persistence, + LOCKMODE lockmode) +{ + Relation rel; + Relation classRel; + HeapTuple tuple, + newtuple; + Datum new_val[Natts_pg_class]; + bool new_null[Natts_pg_class], + new_repl[Natts_pg_class]; + List *relids; + ListCell *lc_oid; + + Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE); + Assert(lockmode == AccessExclusiveLock); + + /* + * Use ATRewriteTable instead of this function if the following condition + * is not satisfied. + */ + Assert(tab->constraints == NULL && tab->partition_constraint == NULL && + tab->newvals == NULL && !tab->verify_new_notnull); + + rel = table_open(tab->relid, lockmode); + + Assert(rel->rd_rel->relpersistence != persistence); + + elog(DEBUG1, "perform in-place persistence change"); + + /* + * Initially, gather all relations that require a persistence change. + */ + + /* Collect OIDs of indexes and toast relations */ + relids = RelationGetIndexList(rel); + relids = lcons_oid(rel->rd_id, relids); + + /* Add toast relation if any */ + if (OidIsValid(rel->rd_rel->reltoastrelid)) + { + List *toastidx; + Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode); + + relids = lappend_oid(relids, rel->rd_rel->reltoastrelid); + toastidx = RelationGetIndexList(toastrel); + relids = list_concat(relids, toastidx); + pfree(toastidx); + table_close(toastrel, NoLock); + } + + table_close(rel, NoLock); + + /* Make changes in storage */ + classRel = table_open(RelationRelationId, RowExclusiveLock); + + foreach(lc_oid, relids) + { + Oid reloid = lfirst_oid(lc_oid); + Relation r = relation_open(reloid, lockmode); + SMgrRelation srel; + bool persistent = (persistence == RELPERSISTENCE_PERMANENT); + bool is_index; + + /* + * Reconstruct the storage when permanent and unlogged storage types + * are incompatible. + */ + if (r->rd_rel->relkind == RELKIND_INDEX && + !r->rd_indam->amunloggedstoragecompatible) + { + int reindex_flags; + ReindexParams params = {0}; + + /* reindex doesn't allow concurrent use of the index */ + table_close(r, NoLock); + + reindex_flags = + REINDEX_REL_SUPPRESS_INDEX_USE | + REINDEX_REL_CHECK_CONSTRAINTS; + + /* Set the same persistence with the parent relation. */ + if (persistent) + reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; + else + reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED; + + /* this doesn't fire REINDEX event triegger */ + reindex_index(NULL, reloid, reindex_flags, persistence, ¶ms); + + continue; + } + + /* Currently, only allowing changes to UNLOGGED. */ + Assert(!persistent); + + RelationAssumePersistenceChange(r); + + /* switch buffer persistence */ + srel = RelationGetSmgr(r); + log_smgrbufpersistence(srel->smgr_rlocator.locator, persistent); + SetRelationBuffersPersistence(srel, persistent); + + /* then create the init fork */ + is_index = (r->rd_rel->relkind == RELKIND_INDEX); + RelationCreateFork(srel, INIT_FORKNUM, !is_index, true); + if (is_index) + r->rd_indam->ambuildempty(r); + + /* Update catalog */ + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", reloid); + + memset(new_val, 0, sizeof(new_val)); + memset(new_null, false, sizeof(new_null)); + memset(new_repl, false, sizeof(new_repl)); + + new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence); + new_null[Anum_pg_class_relpersistence - 1] = false; + new_repl[Anum_pg_class_relpersistence - 1] = true; + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel), + new_val, new_null, new_repl); + + CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple); + heap_freetuple(newtuple); + + table_close(r, NoLock); + } + + table_close(classRel, NoLock); +} + /* * ATRewriteTables: ALTER TABLE phase 3 */ @@ -5876,48 +6013,59 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode, tab->relid, tab->rewrite); - /* - * Create transient table that will receive the modified data. - * - * Ensure it is marked correctly as logged or unlogged. We have - * to do this here so that buffers for the new relfilenumber will - * have the right persistence set, and at the same time ensure - * that the original filenumbers's buffers will get read in with - * the correct setting (i.e. the original one). Otherwise a - * rollback after the rewrite would possibly result with buffers - * for the original filenumbers having the wrong persistence - * setting. - * - * NB: This relies on swap_relation_files() also swapping the - * persistence. That wouldn't work for pg_class, but that can't be - * unlogged anyway. - */ - OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod, - persistence, lockmode); + if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE && + persistence == RELPERSISTENCE_UNLOGGED) + { + /* Make in-place persistence change. */ + RelationChangePersistence(tab, persistence, lockmode); + } + else + { + /* + * Create transient table that will receive the modified data. + * + * Ensure it is marked correctly as logged or unlogged. We + * have to do this here so that buffers for the new + * relfilenumber will have the right persistence set, and at + * the same time ensure that the original filenumbers's buffers + * will get read in with the correct setting (i.e. the original + * one). Otherwise a rollback after the rewrite would possibly + * result with buffers for the original filenumbers having the + * wrong persistence setting. + * + * NB: This relies on swap_relation_files() also swapping the + * persistence. That wouldn't work for pg_class, but that can't + * be unlogged anyway. + */ + OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, + NewAccessMethod, + persistence, lockmode); - /* - * Copy the heap data into the new table with the desired - * modifications, and test the current data within the table - * against new constraints generated by ALTER TABLE commands. - */ - ATRewriteTable(tab, OIDNewHeap, lockmode); + /* + * Copy the heap data into the new table with the desired + * modifications, and test the current data within the table + * against new constraints generated by ALTER TABLE commands. + */ + ATRewriteTable(tab, OIDNewHeap, lockmode); - /* - * Swap the physical files of the old and new heaps, then rebuild - * indexes and discard the old heap. We can use RecentXmin for - * the table's new relfrozenxid because we rewrote all the tuples - * in ATRewriteTable, so no older Xid remains in the table. Also, - * we never try to swap toast tables by content, since we have no - * interest in letting this code work on system catalogs. - */ - finish_heap_swap(tab->relid, OIDNewHeap, - false, false, true, - !OidIsValid(tab->newTableSpace), - RecentXmin, - ReadNextMultiXactId(), - persistence); + /* + * Swap the physical files of the old and new heaps, then + * rebuild indexes and discard the old heap. We can use + * RecentXmin for the table's new relfrozenxid because we + * rewrote all the tuples in ATRewriteTable, so no older Xid + * remains in the table. Also, we never try to swap toast + * tables by content, since we have no interest in letting this + * code work on system catalogs. + */ + finish_heap_swap(tab->relid, OIDNewHeap, + false, false, true, + !OidIsValid(tab->newTableSpace), + RecentXmin, + ReadNextMultiXactId(), + persistence); - InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + } } else if (tab->rewrite > 0 && tab->relkind == RELKIND_SEQUENCE) { diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h index 19badc852a0..857a845eb9d 100644 --- a/src/include/access/undolog.h +++ b/src/include/access/undolog.h @@ -80,7 +80,7 @@ extern Size UndoLogShmemSize(void); extern void UndoLogShmemInit(void); extern void InitUndoLog(void); extern void UndoLogWrite(RmgrId rmgr, uint8 info, void *data, int len); -extern void AtEOXact_UndoLog(TransactionId xid); +extern void AtEOXact_UndoLog(bool isCommit, TransactionId xid); extern void AtPrepare_UndoLog(void); extern void UndoLog_UndoByXid(bool isCommit, TransactionId xid, int nchildren, TransactionId *children); -- 2.43.5 ----Next_Part(Fri_Dec_27_17_25_02_2024_357)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v36-0015-Add-test-for-ALTER-TABLE-UNLOGGED.patch" ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v34 10/16] In-place persistance change to UNLOGGED @ 2024-08-27 02:19 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Kyotaro Horiguchi @ 2024-08-27 02:19 UTC (permalink / raw) This commit enables changing the persistence of relations to UNLOGGED without creating a new storage file. ALTER TABLE LOGGED will continue to create a new storage as before. --- src/backend/commands/tablecmds.c | 226 +++++++++++++++++++++++++------ 1 file changed, 187 insertions(+), 39 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index e9bba3aceb..05364365a7 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5634,6 +5634,143 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, return newcmd; } +/* + * RelationChangePersistence: perform in-place persistence change of a relation + */ +static void +RelationChangePersistence(AlteredTableInfo *tab, char persistence, + LOCKMODE lockmode) +{ + Relation rel; + Relation classRel; + HeapTuple tuple, + newtuple; + Datum new_val[Natts_pg_class]; + bool new_null[Natts_pg_class], + new_repl[Natts_pg_class]; + List *relids; + ListCell *lc_oid; + + Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE); + Assert(lockmode == AccessExclusiveLock); + + /* + * Use ATRewriteTable instead of this function if the following condition + * is not satisfied. + */ + Assert(tab->constraints == NULL && tab->partition_constraint == NULL && + tab->newvals == NULL && !tab->verify_new_notnull); + + rel = table_open(tab->relid, lockmode); + + Assert(rel->rd_rel->relpersistence != persistence); + + elog(DEBUG1, "perform in-place persistence change"); + + /* + * Initially, gather all relations that require a persistence change. + */ + + /* Collect OIDs of indexes and toast relations */ + relids = RelationGetIndexList(rel); + relids = lcons_oid(rel->rd_id, relids); + + /* Add toast relation if any */ + if (OidIsValid(rel->rd_rel->reltoastrelid)) + { + List *toastidx; + Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode); + + relids = lappend_oid(relids, rel->rd_rel->reltoastrelid); + toastidx = RelationGetIndexList(toastrel); + relids = list_concat(relids, toastidx); + pfree(toastidx); + table_close(toastrel, NoLock); + } + + table_close(rel, NoLock); + + /* Make changes in storage */ + classRel = table_open(RelationRelationId, RowExclusiveLock); + + foreach(lc_oid, relids) + { + Oid reloid = lfirst_oid(lc_oid); + Relation r = relation_open(reloid, lockmode); + SMgrRelation srel; + bool persistent = (persistence == RELPERSISTENCE_PERMANENT); + bool is_index; + + /* + * Reconstruct the storage when permanent and unlogged storage types + * are incompatible. + */ + if (r->rd_rel->relkind == RELKIND_INDEX && + !r->rd_indam->amunloggedstoragecompatible) + { + int reindex_flags; + ReindexParams params = {0}; + + /* reindex doesn't allow concurrent use of the index */ + table_close(r, NoLock); + + reindex_flags = + REINDEX_REL_SUPPRESS_INDEX_USE | + REINDEX_REL_CHECK_CONSTRAINTS; + + /* Set the same persistence with the parent relation. */ + if (persistent) + reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; + else + reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED; + + /* this doesn't fire REINDEX event triegger */ + reindex_index(NULL, reloid, reindex_flags, persistence, ¶ms); + + continue; + } + + /* Currently, only allowing changes to UNLOGGED. */ + Assert(!persistent); + + RelationAssumePersistenceChange(r); + + /* switch buffer persistence */ + srel = RelationGetSmgr(r); + log_smgrbufpersistence(srel->smgr_rlocator.locator, persistent); + SetRelationBuffersPersistence(srel, persistent); + + /* then create the init fork */ + is_index = (r->rd_rel->relkind == RELKIND_INDEX); + RelationCreateFork(srel, INIT_FORKNUM, !is_index, true); + if (is_index) + r->rd_indam->ambuildempty(r); + + /* Update catalog */ + tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", reloid); + + memset(new_val, 0, sizeof(new_val)); + memset(new_null, false, sizeof(new_null)); + memset(new_repl, false, sizeof(new_repl)); + + new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence); + new_null[Anum_pg_class_relpersistence - 1] = false; + new_repl[Anum_pg_class_relpersistence - 1] = true; + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel), + new_val, new_null, new_repl); + + CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple); + heap_freetuple(newtuple); + + table_close(r, NoLock); + } + + table_close(classRel, NoLock); +} + /* * ATRewriteTables: ALTER TABLE phase 3 */ @@ -5766,48 +5903,59 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode, tab->relid, tab->rewrite); - /* - * Create transient table that will receive the modified data. - * - * Ensure it is marked correctly as logged or unlogged. We have - * to do this here so that buffers for the new relfilenumber will - * have the right persistence set, and at the same time ensure - * that the original filenumbers's buffers will get read in with - * the correct setting (i.e. the original one). Otherwise a - * rollback after the rewrite would possibly result with buffers - * for the original filenumbers having the wrong persistence - * setting. - * - * NB: This relies on swap_relation_files() also swapping the - * persistence. That wouldn't work for pg_class, but that can't be - * unlogged anyway. - */ - OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, NewAccessMethod, - persistence, lockmode); + if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE && + persistence == RELPERSISTENCE_UNLOGGED) + { + /* Make in-place persistence change. */ + RelationChangePersistence(tab, persistence, lockmode); + } + else + { + /* + * Create transient table that will receive the modified data. + * + * Ensure it is marked correctly as logged or unlogged. We + * have to do this here so that buffers for the new + * relfilenumber will have the right persistence set, and at + * the same time ensure that the original filenumbers's buffers + * will get read in with the correct setting (i.e. the original + * one). Otherwise a rollback after the rewrite would possibly + * result with buffers for the original filenumbers having the + * wrong persistence setting. + * + * NB: This relies on swap_relation_files() also swapping the + * persistence. That wouldn't work for pg_class, but that can't + * be unlogged anyway. + */ + OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, + NewAccessMethod, + persistence, lockmode); - /* - * Copy the heap data into the new table with the desired - * modifications, and test the current data within the table - * against new constraints generated by ALTER TABLE commands. - */ - ATRewriteTable(tab, OIDNewHeap, lockmode); + /* + * Copy the heap data into the new table with the desired + * modifications, and test the current data within the table + * against new constraints generated by ALTER TABLE commands. + */ + ATRewriteTable(tab, OIDNewHeap, lockmode); - /* - * Swap the physical files of the old and new heaps, then rebuild - * indexes and discard the old heap. We can use RecentXmin for - * the table's new relfrozenxid because we rewrote all the tuples - * in ATRewriteTable, so no older Xid remains in the table. Also, - * we never try to swap toast tables by content, since we have no - * interest in letting this code work on system catalogs. - */ - finish_heap_swap(tab->relid, OIDNewHeap, - false, false, true, - !OidIsValid(tab->newTableSpace), - RecentXmin, - ReadNextMultiXactId(), - persistence); + /* + * Swap the physical files of the old and new heaps, then + * rebuild indexes and discard the old heap. We can use + * RecentXmin for the table's new relfrozenxid because we + * rewrote all the tuples in ATRewriteTable, so no older Xid + * remains in the table. Also, we never try to swap toast + * tables by content, since we have no interest in letting this + * code work on system catalogs. + */ + finish_heap_swap(tab->relid, OIDNewHeap, + false, false, true, + !OidIsValid(tab->newTableSpace), + RecentXmin, + ReadNextMultiXactId(), + persistence); - InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + InvokeObjectPostAlterHook(RelationRelationId, tab->relid, 0); + } } else if (tab->rewrite > 0 && tab->relkind == RELKIND_SEQUENCE) { -- 2.43.5 ----Next_Part(Sun_Sep__1_01_09_25_2024_581)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v34-0011-Add-test-for-ALTER-TABLE-UNLOGGED.patch" ^ permalink raw reply [nested|flat] 4+ messages in thread
end of thread, other threads:[~2024-08-27 02:19 UTC | newest] Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-10 09:04 [PATCH 7/8] Alter table set compression Dilip Kumar <[email protected]> 2024-08-27 02:19 [PATCH v35 15/21] In-place persistance change to UNLOGGED Kyotaro Horiguchi <[email protected]> 2024-08-27 02:19 [PATCH v36 14/17] In-place persistance change to UNLOGGED Kyotaro Horiguchi <[email protected]> 2024-08-27 02:19 [PATCH v34 10/16] In-place persistance change to UNLOGGED Kyotaro Horiguchi <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox